diff --git a/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv b/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv index 83a394f9a9755064af42e52717145e8da49e9ec1..900ace3a36f50eca2e7d43d6ac82100d58a639f9 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv +++ b/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv @@ -94,3 +94,107 @@ 5, 59227 6, 59227 7, 59227 +0, 59719 +1, 59151 +2, 59151 +3, 59851 +4, 59851 +5, 59851 +6, 59851 +7, 59851 +0, 60421 +1, 59851 +2, 59153 +3, 59853 +4, 59853 +5, 60551 +6, 60551 +7, 60551 +0, 87601 +1, 84991 +2, 84095 +3, 83869 +4, 84541 +5, 88743 +6, 84651 +7, 89267 +0, 118047 +1, 118053 +2, 118045 +3, 118067 +4, 118063 +5, 118073 +6, 118061 +7, 118053 +0, 118939 +1, 118985 +2, 118967 +3, 119001 +4, 119001 +5, 119001 +6, 119001 +7, 119001 +0, 118939 +1, 118985 +2, 118967 +3, 119001 +4, 119001 +5, 119001 +6, 119001 +7, 119001 +0, 119601 +1, 119645 +2, 119627 +3, 119661 +4, 119661 +5, 119661 +6, 119661 +7, 119661 +0, 128697 +1, 128471 +2, 128525 +3, 128529 +4, 128495 +5, 128527 +6, 128505 +7, 128547 +0, 141327 +1, 141555 +2, 141529 +3, 141355 +4, 141355 +5, 141355 +6, 141355 +7, 141355 +0, 141327 +1, 141555 +2, 141529 +3, 141355 +4, 141355 +5, 141355 +6, 141355 +7, 141355 +0, 140681 +1, 140909 +2, 140947 +3, 141015 +4, 141029 +5, 140709 +6, 140709 +7, 140709 +0, 138373 +1, 138989 +2, 138639 +3, 139435 +4, 139109 +5, 138401 +6, 138789 +7, 138789 +0, 12601 +1, 12604 +2, 12796 +3, 139433 +4, 13564 +5, 12604 +6, 12604 +7, 12604 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log b/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log index 6e51fdbf41d1ebd52d962d4f6747432915a9a279..cae3a9de12467cd6380e7e406c6fb9908fff9541 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log +++ b/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log @@ -993,3 +993,2548 @@ INFO:moonep.api:MoonEP comm buffer sizes: meta_buffer_size : 2.00 MiB local_temp_buffer_size : 1.26 MiB total_size : 241.26 MiB +WARNING:megatron.core.rerun_state_machine:Result validation enabled + [2026-08-26 18:09:50] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22137.0 | throughput per GPU (TFLOP/s/GPU): 5.6 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097850E+01 | loss scale: 1.0 | grad norm: 29.823 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 20.11 +Number of parameters in embedding layers in billions: 0.36 +Total number of parameters in billions: 20.47 +Number of parameters in most loaded shard in billions: 20.4724 +Theoretical memory footprints: weight and optimizer=146430.14 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 104985.1162109375 | max allocated: 104985.11669921875 | reserved: 108632.0 | max reserved: 108632.0 + [2026-08-26 18:09:51] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 413.4 | throughput per GPU (TFLOP/s/GPU): 298.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097848E+01 | loss scale: 1.0 | grad norm: 30.556 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45564 payload_rows_remote=39624 dup_rows=19972 dup_rows_remote=17142 dedup_ratio=0.304749 router_tokens_per_expert=[507, 200, 358, 17, 30, 30, 1113, 23, 1044, 946, 187, 107, 1316, 144, 104, 429, 56, 56, 458, 562, 663, 22, 183, 56, 445, 1129, 369, 822, 1011, 424, 335, 51, 512, 278, 1274, 429, 685, 47, 166, 481, 531, 1239, 800, 151, 974, 263, 61, 366, 1519, 731, 212, 50, 604, 949, 2508, 1099, 410, 648, 279, 51, 543, 1211, 141, 359] payload_rows_per_dst=[5508, 5982, 6396, 5890, 5356, 5940, 4330, 6162] dup_rows_per_dst=[2086, 2572, 2732, 3282, 2388, 2830, 762, 3320] combine_rows_per_src=[5508, 5982, 6396, 5890, 5356, 5940, 4330, 6162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46426 payload_rows_remote=40854 dup_rows=19110 dup_rows_remote=17328 dedup_ratio=0.291595 router_tokens_per_expert=[366, 132, 570, 35, 29, 19, 1055, 15, 1235, 934, 217, 95, 1213, 218, 268, 300, 49, 37, 666, 512, 456, 93, 251, 26, 205, 1099, 341, 1056, 840, 991, 211, 53, 523, 98, 1510, 593, 634, 13, 71, 235, 418, 624, 1218, 61, 661, 149, 298, 360, 1249, 726, 119, 51, 717, 875, 3160, 1771, 304, 444, 286, 261, 814, 721, 40, 177] payload_rows_per_dst=[5270, 6228, 7172, 6198, 5572, 5370, 5018, 5598] dup_rows_per_dst=[1670, 2732, 3328, 3394, 1782, 2208, 1480, 2516] combine_rows_per_src=[5270, 6228, 7172, 6198, 5572, 5370, 5018, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46646 payload_rows_remote=39936 dup_rows=18890 dup_rows_remote=15818 dedup_ratio=0.288239 router_tokens_per_expert=[681, 187, 503, 8, 121, 20, 983, 71, 709, 1273, 281, 69, 1371, 213, 164, 477, 91, 17, 548, 248, 383, 110, 194, 28, 187, 923, 391, 898, 503, 350, 189, 54, 521, 426, 1121, 404, 1033, 13, 69, 238, 692, 597, 1083, 104, 830, 217, 51, 202, 2317, 386, 174, 24, 424, 1248, 3223, 1798, 443, 380, 188, 137, 1004, 864, 88, 224] payload_rows_per_dst=[6710, 6340, 7182, 5298, 5328, 4862, 5974, 4952] dup_rows_per_dst=[3072, 2950, 2502, 2380, 2322, 1826, 2134, 1704] combine_rows_per_src=[6710, 6340, 7182, 5298, 5328, 4862, 5974, 4952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43530 payload_rows_remote=37954 dup_rows=22006 dup_rows_remote=19870 dedup_ratio=0.335785 router_tokens_per_expert=[380, 165, 991, 19, 133, 10, 1068, 185, 887, 1098, 199, 290, 901, 165, 350, 394, 91, 30, 499, 265, 236, 72, 178, 75, 257, 1360, 401, 860, 498, 893, 281, 31, 306, 135, 1413, 282, 1042, 26, 172, 553, 447, 681, 1489, 167, 673, 148, 129, 430, 1261, 678, 135, 27, 1075, 1894, 2487, 1272, 363, 524, 190, 68, 507, 644, 85, 203] payload_rows_per_dst=[5796, 5020, 2582, 4920, 6166, 5768, 7702, 5576] dup_rows_per_dst=[2274, 1746, 310, 1522, 3494, 2560, 7964, 2136] combine_rows_per_src=[5796, 5020, 2582, 4920, 6166, 5768, 7702, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45176 payload_rows_remote=39120 dup_rows=20360 dup_rows_remote=17962 dedup_ratio=0.310669 router_tokens_per_expert=[400, 142, 594, 10, 166, 20, 913, 96, 1159, 638, 271, 276, 1361, 160, 51, 411, 161, 69, 293, 225, 921, 161, 103, 24, 245, 1172, 346, 711, 449, 542, 314, 30, 1082, 660, 1166, 482, 724, 36, 107, 739, 691, 615, 1215, 78, 805, 214, 188, 222, 1238, 387, 176, 38, 359, 1219, 3266, 1434, 232, 460, 221, 59, 815, 614, 78, 714] payload_rows_per_dst=[5402, 5176, 4966, 5154, 6606, 5584, 6056, 6232] dup_rows_per_dst=[1756, 2080, 1772, 2076, 4784, 2472, 2398, 3022] combine_rows_per_src=[5402, 5176, 4966, 5154, 6606, 5584, 6056, 6232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46052 payload_rows_remote=40284 dup_rows=19484 dup_rows_remote=17022 dedup_ratio=0.297302 router_tokens_per_expert=[744, 160, 909, 7, 157, 6, 900, 107, 969, 1237, 215, 119, 922, 234, 85, 334, 77, 68, 365, 196, 356, 77, 568, 58, 661, 1463, 327, 773, 326, 708, 211, 128, 878, 246, 577, 262, 639, 195, 80, 740, 418, 1146, 646, 94, 800, 295, 61, 282, 1905, 280, 100, 21, 851, 1591, 2709, 1021, 380, 369, 174, 74, 428, 1546, 128, 365] payload_rows_per_dst=[6534, 5768, 6540, 6026, 5134, 5500, 5656, 4894] dup_rows_per_dst=[3256, 2462, 2408, 3168, 2100, 1984, 2072, 2034] combine_rows_per_src=[6534, 5768, 6540, 6026, 5134, 5500, 5656, 4894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45844 payload_rows_remote=40412 dup_rows=19692 dup_rows_remote=17642 dedup_ratio=0.300476 router_tokens_per_expert=[693, 161, 1152, 13, 156, 1, 736, 76, 929, 868, 190, 197, 1271, 228, 32, 371, 128, 19, 388, 368, 786, 31, 220, 47, 327, 640, 350, 779, 710, 535, 342, 58, 667, 391, 728, 321, 664, 48, 143, 487, 558, 993, 1207, 210, 1237, 205, 72, 209, 1441, 366, 218, 75, 462, 836, 2454, 808, 648, 834, 454, 108, 646, 1637, 185, 654] payload_rows_per_dst=[6102, 5706, 6242, 5432, 4940, 6400, 4528, 6494] dup_rows_per_dst=[2756, 2466, 2640, 2050, 1958, 2982, 1002, 3838] combine_rows_per_src=[6102, 5706, 6242, 5432, 4940, 6400, 4528, 6494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46172 payload_rows_remote=39836 dup_rows=19364 dup_rows_remote=16936 dedup_ratio=0.295471 router_tokens_per_expert=[371, 114, 962, 24, 158, 41, 367, 114, 662, 1062, 395, 333, 855, 188, 186, 473, 63, 61, 524, 189, 517, 101, 251, 49, 337, 1244, 650, 955, 410, 408, 281, 88, 533, 195, 940, 560, 784, 49, 61, 681, 346, 1078, 1272, 218, 1003, 130, 286, 292, 1521, 452, 99, 14, 837, 1579, 2627, 1249, 334, 521, 280, 111, 561, 1385, 87, 250] payload_rows_per_dst=[5542, 5778, 6336, 5832, 5336, 6228, 6036, 5084] dup_rows_per_dst=[1802, 2530, 2428, 2914, 2270, 3022, 2424, 1974] combine_rows_per_src=[5542, 5778, 6336, 5832, 5336, 6228, 6036, 5084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286261248 dispatch_meta_bytes=223016 combine_payload_bytes=286261248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=286484264 ingress_bytes=396361728 link_bytes=396361728 token_bytes=572745512 weight_movement_bytes=110100480 total_nvlink_bytes=682845992 weight_to_token_ratio=0.192233 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.893382353e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=292841472 dispatch_meta_bytes=232728 combine_payload_bytes=292841472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293074200 ingress_bytes=402941952 link_bytes=402941952 token_bytes=585915672 weight_movement_bytes=110100480 total_nvlink_bytes=696016152 weight_to_token_ratio=0.187912 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.007823271e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284024832 dispatch_meta_bytes=227064 combine_payload_bytes=284024832 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=284251896 ingress_bytes=504225792 link_bytes=504225792 token_bytes=568276728 weight_movement_bytes=220200960 total_nvlink_bytes=788477688 weight_to_token_ratio=0.387489 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=8576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.769315832e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=272054272 dispatch_meta_bytes=231296 combine_payload_bytes=272054272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=272285568 ingress_bytes=382154752 link_bytes=382154752 token_bytes=544339840 weight_movement_bytes=110100480 total_nvlink_bytes=654440320 weight_to_token_ratio=0.202264 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.646299674e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280412160 dispatch_meta_bytes=228328 combine_payload_bytes=280412160 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=280640488 ingress_bytes=720814080 link_bytes=720814080 token_bytes=561052648 weight_movement_bytes=440401920 total_nvlink_bytes=1001454568 weight_to_token_ratio=0.784956 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.253614239e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=288755712 dispatch_meta_bytes=229224 combine_payload_bytes=288755712 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=288984936 ingress_bytes=508956672 link_bytes=508956672 token_bytes=577740648 weight_movement_bytes=220200960 total_nvlink_bytes=797941608 weight_to_token_ratio=0.381142 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.851593616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=289673216 dispatch_meta_bytes=232216 combine_payload_bytes=289673216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=289905432 ingress_bytes=509874176 link_bytes=509874176 token_bytes=579578648 weight_movement_bytes=220200960 total_nvlink_bytes=799779608 weight_to_token_ratio=0.379933 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.867550520e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=285544448 dispatch_meta_bytes=227088 combine_payload_bytes=285544448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=285771536 ingress_bytes=395644928 link_bytes=395644928 token_bytes=571315984 weight_movement_bytes=110100480 total_nvlink_bytes=681416464 weight_to_token_ratio=0.192714 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.880916022e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46430 payload_rows_remote=40964 dup_rows=19106 dup_rows_remote=17624 dedup_ratio=0.291534 router_tokens_per_expert=[2, 1, 677, 91, 11, 22, 862, 1, 921, 1488, 155, 12, 1745, 985, 5, 1379, 1, 240, 98, 380, 1, 447, 9, 1052, 126, 269, 2, 1129, 2422, 132, 32, 13, 3195, 827, 0, 219, 358, 14, 15, 0, 61, 0, 1466, 85, 625, 0, 4, 130, 1907, 2375, 359, 927, 217, 144, 57, 1, 13, 10, 1329, 46, 907, 1807, 480, 480] payload_rows_per_dst=[5466, 7096, 5518, 7512, 5104, 4862, 5742, 5130] dup_rows_per_dst=[1482, 4690, 1696, 5488, 1098, 1850, 1402, 1400] combine_rows_per_src=[5466, 7096, 5518, 7512, 5104, 4862, 5742, 5130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47760 payload_rows_remote=40470 dup_rows=17776 dup_rows_remote=16006 dedup_ratio=0.271240 router_tokens_per_expert=[5, 32, 1732, 837, 17, 32, 542, 28, 972, 697, 312, 27, 2016, 3078, 19, 1804, 0, 83, 36, 935, 42, 221, 15, 809, 64, 321, 7, 838, 1209, 166, 73, 41, 3498, 738, 1, 35, 203, 7, 39, 9, 2, 19, 1320, 105, 148, 1, 0, 259, 1756, 1198, 93, 347, 271, 267, 29, 88, 31, 146, 487, 49, 2331, 1994, 125, 162] payload_rows_per_dst=[6582, 5014, 6146, 4284, 7290, 7228, 5724, 5492] dup_rows_per_dst=[3856, 1286, 3262, 1154, 1770, 2904, 2374, 1170] combine_rows_per_src=[6582, 5014, 6146, 4284, 7290, 7228, 5724, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44382 payload_rows_remote=36928 dup_rows=21154 dup_rows_remote=14228 dedup_ratio=0.322784 router_tokens_per_expert=[16, 13, 1167, 333, 43, 26, 452, 21, 1030, 660, 212, 18, 1188, 1896, 29, 1778, 0, 364, 19, 471, 21, 484, 34, 883, 149, 505, 4, 1149, 1422, 203, 16, 30, 2358, 812, 4, 92, 120, 3, 104, 47, 9, 7, 1611, 182, 347, 0, 2, 165, 1390, 1856, 308, 421, 853, 1329, 59, 51, 47, 110, 752, 146, 1703, 2592, 133, 519] payload_rows_per_dst=[3644, 3506, 5966, 5136, 5754, 5890, 7032, 7454] dup_rows_per_dst=[498, 392, 2142, 1820, 1326, 2548, 5502, 6926] combine_rows_per_src=[3644, 3506, 5966, 5136, 5754, 5890, 7032, 7454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47354 payload_rows_remote=39994 dup_rows=18182 dup_rows_remote=13796 dedup_ratio=0.277435 router_tokens_per_expert=[69, 59, 1048, 168, 30, 23, 598, 4, 1651, 1537, 344, 13, 1885, 2955, 29, 2132, 1, 108, 86, 996, 1, 164, 0, 686, 141, 686, 15, 558, 1710, 45, 63, 18, 2449, 930, 0, 175, 228, 62, 6, 18, 16, 9, 770, 152, 324, 3, 12, 170, 1113, 1933, 739, 65, 555, 297, 135, 24, 31, 67, 1455, 21, 416, 1838, 387, 545] payload_rows_per_dst=[5622, 5524, 6322, 4954, 6008, 7360, 6358, 5206] dup_rows_per_dst=[2052, 1624, 2026, 1518, 1728, 4386, 3364, 1484] combine_rows_per_src=[5622, 5524, 6322, 4954, 6008, 7360, 6358, 5206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=293629952 dispatch_meta_bytes=234352 combine_payload_bytes=293629952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293864304 ingress_bytes=403730432 link_bytes=403730432 token_bytes=587494256 weight_movement_bytes=110100480 total_nvlink_bytes=697594736 weight_to_token_ratio=0.187407 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.021536235e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46740 payload_rows_remote=40824 dup_rows=18796 dup_rows_remote=16386 dedup_ratio=0.286804 router_tokens_per_expert=[77, 6, 796, 394, 85, 37, 326, 4, 1542, 1463, 342, 0, 1867, 1730, 25, 1594, 0, 410, 34, 621, 1, 920, 17, 632, 108, 352, 2, 1139, 1340, 205, 3, 139, 3240, 739, 1, 196, 46, 0, 25, 119, 4, 8, 739, 70, 348, 1, 11, 273, 1328, 1518, 146, 653, 357, 75, 70, 16, 51, 43, 1786, 109, 1613, 2334, 207, 431] payload_rows_per_dst=[5454, 5278, 5870, 5018, 6992, 4930, 5916, 7282] dup_rows_per_dst=[1944, 1466, 2588, 1558, 1740, 1438, 2410, 5652] combine_rows_per_src=[5454, 5278, 5870, 5018, 6992, 4930, 5916, 7282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48722 payload_rows_remote=41722 dup_rows=16814 dup_rows_remote=13346 dedup_ratio=0.256561 router_tokens_per_expert=[35, 10, 1740, 284, 40, 35, 691, 52, 1573, 1541, 330, 15, 1711, 1736, 64, 2013, 1, 83, 107, 376, 25, 637, 7, 374, 308, 75, 50, 482, 1929, 44, 12, 17, 3509, 677, 78, 21, 255, 77, 250, 5, 50, 2, 678, 308, 188, 19, 38, 251, 1934, 1420, 156, 62, 219, 956, 116, 48, 11, 25, 1970, 57, 349, 2389, 110, 143] payload_rows_per_dst=[6778, 7000, 5802, 6408, 7350, 5000, 5594, 4790] dup_rows_per_dst=[3774, 3468, 1444, 2266, 2394, 1540, 1388, 540] combine_rows_per_src=[6778, 7000, 5802, 6408, 7350, 5000, 5594, 4790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48654 payload_rows_remote=41812 dup_rows=16882 dup_rows_remote=13662 dedup_ratio=0.257599 router_tokens_per_expert=[78, 58, 1294, 399, 4, 88, 396, 1, 966, 668, 318, 90, 2714, 1971, 9, 2651, 1, 180, 78, 410, 3, 77, 7, 874, 192, 278, 3, 1250, 1558, 74, 7, 16, 2973, 738, 0, 137, 183, 8, 187, 13, 5, 1, 573, 70, 323, 0, 0, 463, 1311, 1859, 951, 49, 396, 98, 20, 6, 26, 24, 2087, 71, 1015, 1689, 353, 426] payload_rows_per_dst=[5738, 6800, 6534, 6842, 6560, 5252, 4788, 6140] dup_rows_per_dst=[2276, 2730, 2028, 3220, 1918, 1560, 1286, 1864] combine_rows_per_src=[5738, 6800, 6534, 6842, 6560, 5252, 4788, 6140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=290088960 dispatch_meta_bytes=225904 combine_payload_bytes=290088960 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=290314864 ingress_bytes=510289920 link_bytes=510289920 token_bytes=580403824 weight_movement_bytes=220200960 total_nvlink_bytes=800604784 weight_to_token_ratio=0.379393 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.874780991e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264699904 dispatch_meta_bytes=204624 combine_payload_bytes=264699904 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=264904528 ingress_bytes=484900864 link_bytes=484900864 token_bytes=529604432 weight_movement_bytes=220200960 total_nvlink_bytes=749805392 weight_to_token_ratio=0.415784 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.433223550e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=286676992 dispatch_meta_bytes=215160 combine_payload_bytes=286676992 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=286892152 ingress_bytes=506877952 link_bytes=506877952 token_bytes=573569144 weight_movement_bytes=220200960 total_nvlink_bytes=793770104 weight_to_token_ratio=0.383914 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=16128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.815441256e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47576 payload_rows_remote=41340 dup_rows=17960 dup_rows_remote=16004 dedup_ratio=0.274048 router_tokens_per_expert=[41, 13, 1608, 487, 28, 43, 437, 77, 1003, 1373, 304, 15, 1978, 1738, 158, 2354, 1, 338, 66, 380, 4, 552, 12, 389, 235, 93, 11, 284, 1155, 331, 67, 99, 2770, 732, 1, 269, 284, 8, 174, 14, 17, 0, 1659, 116, 575, 2, 14, 273, 1765, 1904, 220, 133, 374, 737, 91, 8, 16, 38, 1517, 119, 460, 2451, 102, 251] payload_rows_per_dst=[6558, 6708, 6236, 6238, 6430, 5940, 5098, 4368] dup_rows_per_dst=[3812, 2954, 1956, 2120, 2074, 2848, 1558, 638] combine_rows_per_src=[6558, 6708, 6236, 6238, 6430, 5940, 5098, 4368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=292626432 dispatch_meta_bytes=228840 combine_payload_bytes=292626432 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=292855272 ingress_bytes=512827392 link_bytes=512827392 token_bytes=585481704 weight_movement_bytes=220200960 total_nvlink_bytes=805682664 weight_to_token_ratio=0.376102 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=12544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.918911803e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=299063296 dispatch_meta_bytes=220272 combine_payload_bytes=299063296 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=299283568 ingress_bytes=959666176 link_bytes=959666176 token_bytes=598346864 weight_movement_bytes=660602880 total_nvlink_bytes=1258949744 weight_to_token_ratio=1.104047 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=9728 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.669017318e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=299708416 dispatch_meta_bytes=221896 combine_payload_bytes=299708416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=299930312 ingress_bytes=409808896 link_bytes=409808896 token_bytes=599638728 weight_movement_bytes=110100480 total_nvlink_bytes=709739208 weight_to_token_ratio=0.183611 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.127250721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46594 payload_rows_remote=40866 dup_rows=18942 dup_rows_remote=17114 dedup_ratio=0.289032 router_tokens_per_expert=[1306, 719, 20, 44, 31, 44, 547, 0, 0, 3374, 6, 30, 40, 254, 3039, 0, 0, 713, 0, 198, 173, 2287, 1, 3, 1629, 0, 156, 908, 1, 5, 119, 1108, 440, 1040, 3, 594, 572, 37, 1853, 8, 303, 1027, 7, 3, 808, 2901, 0, 1, 11, 1, 1711, 2, 569, 3, 186, 0, 645, 4, 103, 25, 1702, 0, 1067, 387] payload_rows_per_dst=[5728, 7870, 5568, 4856, 6376, 7128, 4258, 4810] dup_rows_per_dst=[1828, 7432, 1182, 1180, 2718, 2972, 708, 922] combine_rows_per_src=[5728, 7870, 5568, 4856, 6376, 7128, 4258, 4810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=296325120 dispatch_meta_bytes=229376 combine_payload_bytes=296325120 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=296554496 ingress_bytes=516526080 link_bytes=516526080 token_bytes=592879616 weight_movement_bytes=220200960 total_nvlink_bytes=813080576 weight_to_token_ratio=0.371409 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=15744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.983238071e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=292927488 dispatch_meta_bytes=231920 combine_payload_bytes=292927488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293159408 ingress_bytes=403027968 link_bytes=403027968 token_bytes=586086896 weight_movement_bytes=110100480 total_nvlink_bytes=696187376 weight_to_token_ratio=0.187857 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=9600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.009319231e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47002 payload_rows_remote=42866 dup_rows=18534 dup_rows_remote=17726 dedup_ratio=0.282806 router_tokens_per_expert=[445, 2119, 120, 24, 8, 173, 688, 2, 0, 2988, 12, 12, 334, 523, 2720, 0, 0, 248, 0, 1173, 58, 1504, 117, 30, 2126, 9, 1538, 1330, 1, 2, 19, 1156, 418, 380, 5, 134, 312, 0, 1215, 8, 670, 101, 0, 28, 1189, 2103, 0, 0, 36, 0, 1126, 1, 982, 0, 227, 0, 769, 0, 25, 5, 761, 0, 2408, 386] payload_rows_per_dst=[5838, 4224, 6158, 7462, 4136, 5944, 7114, 6126] dup_rows_per_dst=[1426, 760, 2856, 4900, 808, 2238, 3070, 2476] combine_rows_per_src=[5838, 4224, 6158, 7462, 4136, 5944, 7114, 6126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47010 payload_rows_remote=40498 dup_rows=18526 dup_rows_remote=15642 dedup_ratio=0.282684 router_tokens_per_expert=[675, 1685, 46, 6, 10, 251, 350, 3, 18, 2649, 6, 20, 625, 1163, 3197, 3, 0, 543, 0, 83, 31, 2035, 5, 14, 1399, 0, 253, 1871, 0, 15, 23, 1437, 1278, 445, 13, 194, 376, 13, 1580, 2, 386, 436, 11, 73, 1015, 2467, 0, 0, 3, 0, 1540, 0, 764, 0, 635, 3, 465, 7, 21, 17, 889, 0, 1518, 201] payload_rows_per_dst=[5252, 3404, 7284, 6038, 3982, 6928, 7610, 6512] dup_rows_per_dst=[800, 266, 3436, 1990, 740, 3736, 4674, 2884] combine_rows_per_src=[5252, 3404, 7284, 6038, 3982, 6928, 7610, 6512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46908 payload_rows_remote=40722 dup_rows=18628 dup_rows_remote=17134 dedup_ratio=0.284241 router_tokens_per_expert=[1238, 989, 70, 19, 34, 204, 1237, 6, 5, 2046, 10, 2, 145, 198, 3051, 0, 0, 329, 0, 565, 212, 1263, 5, 1, 1423, 0, 196, 1941, 36, 10, 135, 725, 1365, 919, 11, 218, 1130, 2, 1389, 32, 382, 124, 0, 6, 616, 2706, 2, 4, 1, 0, 1774, 0, 498, 3, 210, 1, 803, 3, 16, 34, 854, 1, 2963, 606] payload_rows_per_dst=[6014, 698, 6552, 6614, 6552, 6186, 7366, 6926] dup_rows_per_dst=[1580, 22, 2290, 2318, 3580, 1494, 3710, 3634] combine_rows_per_src=[6014, 698, 6552, 6614, 6552, 6186, 7366, 6926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47094 payload_rows_remote=41120 dup_rows=18442 dup_rows_remote=16604 dedup_ratio=0.281403 router_tokens_per_expert=[1770, 868, 121, 5, 0, 195, 397, 0, 67, 3323, 0, 29, 85, 683, 2189, 1, 0, 283, 15, 236, 17, 2496, 12, 10, 1565, 1, 453, 1014, 0, 16, 41, 948, 281, 899, 57, 115, 118, 0, 3165, 25, 208, 229, 0, 87, 960, 2366, 52, 1, 3, 0, 992, 0, 497, 4, 221, 0, 1464, 2, 3, 8, 1171, 1, 2889, 110] payload_rows_per_dst=[5560, 1668, 7752, 6018, 7062, 6014, 5974, 7046] dup_rows_per_dst=[1152, 62, 5032, 2058, 2180, 1792, 1838, 4328] combine_rows_per_src=[5560, 1668, 7752, 6018, 7062, 6014, 5974, 7046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45834 payload_rows_remote=38016 dup_rows=19702 dup_rows_remote=11810 dedup_ratio=0.300629 router_tokens_per_expert=[1240, 543, 226, 0, 0, 193, 445, 15, 2, 1347, 7, 0, 138, 1592, 3138, 1, 67, 109, 30, 1173, 124, 1099, 3, 33, 1770, 9, 121, 1630, 45, 275, 11, 1057, 533, 1298, 2, 112, 1542, 4, 968, 28, 575, 37, 0, 6, 259, 2436, 8, 0, 11, 8, 1920, 4, 1038, 69, 826, 4, 574, 16, 42, 246, 1104, 0, 1910, 745] payload_rows_per_dst=[6494, 7818, 4388, 5312, 6120, 5474, 5714, 4514] dup_rows_per_dst=[2650, 7892, 888, 1264, 2854, 1168, 2046, 940] combine_rows_per_src=[6494, 7818, 4388, 5312, 6120, 5474, 5714, 4514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47634 payload_rows_remote=41750 dup_rows=17902 dup_rows_remote=16082 dedup_ratio=0.273163 router_tokens_per_expert=[1636, 1386, 28, 18, 3, 105, 555, 0, 1, 1782, 11, 25, 107, 279, 1782, 1, 0, 855, 0, 706, 139, 1784, 0, 0, 1827, 0, 136, 1224, 0, 6, 70, 589, 264, 1481, 1, 263, 113, 2, 1921, 27, 342, 837, 0, 2, 1067, 1831, 1, 0, 0, 0, 1987, 0, 332, 0, 209, 0, 2220, 1, 5, 31, 1658, 0, 2862, 256] payload_rows_per_dst=[7710, 5818, 5354, 5884, 6124, 6108, 4568, 6068] dup_rows_per_dst=[5476, 1920, 1614, 1820, 2020, 2052, 726, 2274] combine_rows_per_src=[7710, 5818, 5354, 5884, 6124, 6108, 4568, 6068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=290289664 dispatch_meta_bytes=224560 combine_payload_bytes=290289664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=290514224 ingress_bytes=510490624 link_bytes=510490624 token_bytes=580803888 weight_movement_bytes=220200960 total_nvlink_bytes=801004848 weight_to_token_ratio=0.379131 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.878271564e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=307263488 dispatch_meta_bytes=242368 combine_payload_bytes=307263488 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=307505856 ingress_bytes=527464448 link_bytes=527464448 token_bytes=614769344 weight_movement_bytes=220200960 total_nvlink_bytes=834970304 weight_to_token_ratio=0.358185 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.173474281e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=291895296 dispatch_meta_bytes=231424 combine_payload_bytes=291895296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=292126720 ingress_bytes=401995776 link_bytes=401995776 token_bytes=584022016 weight_movement_bytes=110100480 total_nvlink_bytes=694122496 weight_to_token_ratio=0.188521 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.991367714e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=294748160 dispatch_meta_bytes=230896 combine_payload_bytes=294748160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=294979056 ingress_bytes=404848640 link_bytes=404848640 token_bytes=589727216 weight_movement_bytes=110100480 total_nvlink_bytes=699827696 weight_to_token_ratio=0.186697 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.040983711e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45992 payload_rows_remote=41674 dup_rows=19544 dup_rows_remote=18680 dedup_ratio=0.298218 router_tokens_per_expert=[497, 1225, 133, 4, 17, 462, 881, 2, 0, 2532, 28, 3, 92, 960, 3000, 0, 5, 547, 8, 828, 13, 1069, 110, 11, 498, 0, 623, 2793, 14, 29, 36, 1144, 1099, 629, 8, 819, 404, 2, 2149, 32, 320, 137, 0, 4, 476, 2214, 0, 0, 0, 0, 2451, 40, 228, 11, 347, 3, 887, 1, 119, 5, 1251, 0, 793, 775] payload_rows_per_dst=[5876, 7808, 4318, 5612, 6838, 5328, 5350, 4862] dup_rows_per_dst=[2152, 7874, 864, 2210, 3446, 974, 810, 1214] combine_rows_per_src=[5876, 7808, 4318, 5612, 6838, 5328, 5350, 4862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272498688 dispatch_meta_bytes=199304 combine_payload_bytes=272498688 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=272697992 ingress_bytes=602800128 link_bytes=602800128 token_bytes=545196680 weight_movement_bytes=330301440 total_nvlink_bytes=875498120 weight_to_token_ratio=0.605839 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.048368566e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=299264000 dispatch_meta_bytes=231328 combine_payload_bytes=299264000 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=299495328 ingress_bytes=629565440 link_bytes=629565440 token_bytes=598759328 weight_movement_bytes=330301440 total_nvlink_bytes=929060768 weight_to_token_ratio=0.551643 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=11264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.094917846e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46148 payload_rows_remote=38916 dup_rows=19388 dup_rows_remote=16710 dedup_ratio=0.295837 router_tokens_per_expert=[0, 1, 167, 344, 2977, 232, 1207, 27, 17, 0, 0, 73, 3252, 2430, 21, 14, 216, 0, 2532, 159, 54, 2, 8, 0, 22, 80, 2106, 11, 0, 14, 9, 0, 6, 0, 66, 1600, 7, 3822, 298, 310, 0, 2635, 193, 7, 251, 0, 956, 4, 2243, 2, 114, 4, 186, 0, 210, 19, 1527, 30, 19, 40, 0, 0, 59, 2185] payload_rows_per_dst=[7232, 8160, 7366, 4398, 5058, 2578, 5140, 6216] dup_rows_per_dst=[2678, 9300, 3846, 86, 1314, 244, 416, 1504] combine_rows_per_src=[7232, 8160, 7366, 4398, 5058, 2578, 5140, 6216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=298719232 dispatch_meta_bytes=241416 combine_payload_bytes=298719232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298960648 ingress_bytes=408819712 link_bytes=408819712 token_bytes=597679880 weight_movement_bytes=110100480 total_nvlink_bytes=707780360 weight_to_token_ratio=0.184213 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.110047185e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=222504 combine_payload_bytes=278949888 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279172392 ingress_bytes=499150848 link_bytes=499150848 token_bytes=558122280 weight_movement_bytes=220200960 total_nvlink_bytes=778323240 weight_to_token_ratio=0.394539 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=18816 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681054209e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46330 payload_rows_remote=38322 dup_rows=19206 dup_rows_remote=16198 dedup_ratio=0.293060 router_tokens_per_expert=[5, 27, 84, 230, 2067, 10, 2748, 8, 7, 0, 5, 139, 3431, 3401, 128, 5, 638, 9, 2736, 23, 54, 0, 95, 5, 125, 317, 540, 22, 2, 52, 13, 0, 0, 1, 145, 2006, 2, 3971, 526, 75, 1, 3461, 7, 9, 205, 7, 1542, 0, 1599, 12, 146, 7, 371, 0, 154, 58, 228, 118, 4, 0, 2, 1, 119, 1065] payload_rows_per_dst=[3840, 562, 5958, 7224, 5330, 8124, 7284, 8008] dup_rows_per_dst=[232, 6, 1162, 1780, 970, 9142, 2906, 3008] combine_rows_per_src=[3840, 562, 5958, 7224, 5330, 8124, 7284, 8008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46866 payload_rows_remote=39382 dup_rows=18670 dup_rows_remote=16746 dedup_ratio=0.284882 router_tokens_per_expert=[0, 12, 491, 22, 2138, 6, 3313, 8, 1, 1, 0, 526, 2374, 3541, 60, 1, 136, 11, 2548, 46, 65, 0, 18, 2, 37, 84, 634, 12, 6, 18, 336, 6, 20, 0, 91, 1372, 2, 4074, 1102, 3, 0, 3675, 1, 28, 1707, 0, 800, 0, 1247, 5, 74, 8, 50, 0, 5, 2, 53, 3, 0, 4, 0, 1, 183, 1805] payload_rows_per_dst=[4842, 2680, 5394, 5626, 4556, 8104, 7484, 8180] dup_rows_per_dst=[512, 488, 258, 1388, 624, 9410, 1924, 4066] combine_rows_per_src=[4842, 2680, 5394, 5626, 4556, 8104, 7484, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51344 payload_rows_remote=44250 dup_rows=14192 dup_rows_remote=10126 dedup_ratio=0.216553 router_tokens_per_expert=[2, 236, 112, 45, 2042, 7, 1882, 4, 2, 0, 1, 181, 3653, 3854, 14, 2, 287, 3, 1717, 2, 13, 0, 137, 5, 282, 99, 965, 2, 1, 23, 208, 0, 0, 0, 1, 2362, 62, 3677, 576, 9, 0, 3931, 26, 0, 162, 0, 1153, 4, 2552, 1, 50, 0, 19, 0, 337, 5, 104, 162, 1, 0, 2, 0, 52, 1739] payload_rows_per_dst=[4768, 7734, 4034, 7674, 7094, 8006, 6868, 5166] dup_rows_per_dst=[470, 374, 294, 2792, 4066, 2546, 2482, 1168] combine_rows_per_src=[4768, 7734, 4034, 7674, 7094, 8006, 6868, 5166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52738 payload_rows_remote=46166 dup_rows=12798 dup_rows_remote=11456 dedup_ratio=0.195282 router_tokens_per_expert=[0, 19, 88, 182, 2908, 272, 3016, 64, 38, 0, 41, 132, 3288, 3374, 146, 20, 173, 40, 3405, 93, 162, 0, 40, 6, 5, 326, 655, 175, 0, 57, 370, 0, 20, 1, 18, 1097, 4, 3974, 603, 21, 1, 2825, 18, 57, 156, 0, 899, 1, 1880, 5, 276, 8, 173, 0, 187, 5, 328, 135, 17, 3, 4, 1, 78, 878] payload_rows_per_dst=[6260, 6938, 7012, 7194, 3254, 6572, 7422, 8086] dup_rows_per_dst=[806, 564, 826, 2558, 274, 1342, 3678, 2750] combine_rows_per_src=[6260, 6938, 7012, 7194, 3254, 6572, 7422, 8086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49512 payload_rows_remote=42082 dup_rows=16024 dup_rows_remote=11374 dedup_ratio=0.244507 router_tokens_per_expert=[19, 193, 23, 81, 2153, 95, 2519, 10, 85, 2, 35, 35, 3015, 2849, 1, 18, 757, 107, 1412, 23, 15, 1, 2, 10, 0, 32, 2096, 39, 0, 638, 221, 34, 76, 0, 0, 468, 49, 3384, 1186, 136, 3, 1941, 0, 13, 51, 5, 1607, 0, 2982, 23, 527, 109, 6, 1, 545, 1, 120, 127, 6, 87, 22, 13, 14, 2746] payload_rows_per_dst=[7132, 7430, 6314, 5102, 7528, 3328, 6742, 5936] dup_rows_per_dst=[3054, 4650, 2222, 1018, 3070, 30, 1646, 334] combine_rows_per_src=[7132, 7430, 6314, 5102, 7528, 3328, 6742, 5936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50400 payload_rows_remote=42640 dup_rows=15136 dup_rows_remote=10406 dedup_ratio=0.230957 router_tokens_per_expert=[0, 40, 83, 81, 1758, 143, 3536, 2, 6, 0, 0, 28, 3345, 2652, 303, 14, 99, 2, 2480, 51, 37, 0, 40, 7, 6, 156, 2548, 1, 0, 9, 180, 0, 1, 0, 4, 1031, 0, 3614, 552, 54, 0, 3727, 0, 3, 363, 0, 444, 0, 2145, 2, 79, 1, 186, 0, 45, 8, 116, 1, 0, 0, 9, 0, 34, 2742] payload_rows_per_dst=[7790, 5722, 6878, 7760, 7816, 4138, 4682, 5614] dup_rows_per_dst=[3496, 284, 2880, 4730, 2696, 610, 250, 190] combine_rows_per_src=[7790, 5722, 6878, 7760, 7816, 4138, 4682, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=274692096 dispatch_meta_bytes=218080 combine_payload_bytes=274692096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=274910176 ingress_bytes=384792576 link_bytes=384792576 token_bytes=549602272 weight_movement_bytes=110100480 total_nvlink_bytes=659702752 weight_to_token_ratio=0.200328 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.692175772e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=282290176 dispatch_meta_bytes=224512 combine_payload_bytes=282290176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282514688 ingress_bytes=392390656 link_bytes=392390656 token_bytes=564804864 weight_movement_bytes=110100480 total_nvlink_bytes=674905344 weight_to_token_ratio=0.194935 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.824318880e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=317184000 dispatch_meta_bytes=217504 combine_payload_bytes=317184000 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=317401504 ingress_bytes=647485440 link_bytes=647485440 token_bytes=634585504 weight_movement_bytes=330301440 total_nvlink_bytes=964886944 weight_to_token_ratio=0.520500 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.126083673e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=330917888 dispatch_meta_bytes=230488 combine_payload_bytes=330917888 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=331148376 ingress_bytes=551118848 link_bytes=551118848 token_bytes=662066264 weight_movement_bytes=220200960 total_nvlink_bytes=882267224 weight_to_token_ratio=0.332597 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.584863201e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47220 payload_rows_remote=40212 dup_rows=18316 dup_rows_remote=14232 dedup_ratio=0.279480 router_tokens_per_expert=[10, 75, 231, 51, 3145, 4, 2677, 19, 20, 0, 63, 149, 3152, 2974, 144, 165, 863, 2, 2136, 24, 28, 0, 16, 10, 97, 327, 1210, 0, 0, 106, 17, 1, 1, 0, 118, 1697, 0, 3781, 39, 307, 20, 2467, 50, 1, 184, 0, 424, 2, 2805, 10, 85, 0, 52, 0, 234, 120, 489, 30, 0, 0, 40, 0, 6, 2090] payload_rows_per_dst=[7884, 7378, 7008, 4884, 8060, 1284, 5972, 4750] dup_rows_per_dst=[4540, 3266, 4084, 1322, 3826, 78, 640, 560] combine_rows_per_src=[7884, 7378, 7008, 4884, 8060, 1284, 5972, 4750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=301643776 dispatch_meta_bytes=213824 combine_payload_bytes=301643776 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=301857600 ingress_bytes=631945216 link_bytes=631945216 token_bytes=603501376 weight_movement_bytes=330301440 total_nvlink_bytes=933802816 weight_to_token_ratio=0.547309 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.099056668e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50314 payload_rows_remote=44090 dup_rows=15222 dup_rows_remote=13142 dedup_ratio=0.232269 router_tokens_per_expert=[0, 2327, 2, 1, 25, 14, 20, 466, 0, 3, 0, 89, 5, 163, 0, 13, 1004, 55, 1297, 1421, 675, 1449, 43, 2016, 91, 0, 10, 0, 247, 0, 34, 0, 5, 0, 282, 2343, 0, 0, 737, 893, 2, 1910, 423, 80, 3522, 17, 14, 180, 3, 3796, 18, 534, 465, 1, 42, 41, 7, 5, 0, 1319, 7, 679, 19, 3954] payload_rows_per_dst=[6224, 5934, 6082, 7928, 6726, 5736, 8002, 3682] dup_rows_per_dst=[2080, 1486, 2518, 744, 4470, 1736, 1798, 390] combine_rows_per_src=[6224, 5934, 6082, 7928, 6726, 5736, 8002, 3682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=305643520 dispatch_meta_bytes=212184 combine_payload_bytes=305643520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=305855704 ingress_bytes=415744000 link_bytes=415744000 token_bytes=611499224 weight_movement_bytes=110100480 total_nvlink_bytes=721599704 weight_to_token_ratio=0.180050 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.230471941e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=288239616 dispatch_meta_bytes=217776 combine_payload_bytes=288239616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=288457392 ingress_bytes=398340096 link_bytes=398340096 token_bytes=576697008 weight_movement_bytes=110100480 total_nvlink_bytes=686797488 weight_to_token_ratio=0.190916 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.927789426e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45420 payload_rows_remote=37952 dup_rows=20116 dup_rows_remote=11732 dedup_ratio=0.306946 router_tokens_per_expert=[0, 1478, 261, 17, 365, 36, 172, 723, 2, 0, 5, 23, 0, 917, 1, 87, 715, 337, 1973, 464, 583, 319, 208, 2201, 58, 0, 9, 0, 777, 16, 352, 11, 44, 0, 781, 1829, 12, 1, 1064, 1342, 3, 405, 459, 71, 3661, 7, 1, 43, 9, 2752, 0, 698, 96, 16, 158, 222, 1002, 0, 0, 1483, 6, 1538, 0, 2955] payload_rows_per_dst=[4972, 5420, 6276, 3488, 5186, 5778, 6832, 7468] dup_rows_per_dst=[1132, 1052, 2922, 732, 1302, 1268, 3324, 8384] combine_rows_per_src=[4972, 5420, 6276, 3488, 5186, 5778, 6832, 7468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=316037120 dispatch_meta_bytes=228928 combine_payload_bytes=316037120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316266048 ingress_bytes=426137600 link_bytes=426137600 token_bytes=632303168 weight_movement_bytes=110100480 total_nvlink_bytes=742403648 weight_to_token_ratio=0.174126 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.411233740e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47840 payload_rows_remote=42858 dup_rows=17696 dup_rows_remote=17324 dedup_ratio=0.270020 router_tokens_per_expert=[2, 1373, 8, 0, 9, 22, 90, 1305, 0, 0, 1, 111, 0, 1134, 0, 173, 256, 292, 881, 1933, 581, 893, 776, 1832, 161, 3, 12, 1, 906, 1, 92, 0, 652, 0, 374, 3008, 2, 0, 453, 1489, 0, 897, 552, 3, 3263, 0, 0, 9, 9, 2085, 9, 144, 333, 0, 68, 29, 282, 2, 0, 1718, 0, 1778, 60, 2701] payload_rows_per_dst=[4774, 5048, 6856, 6364, 4884, 7234, 4982, 7698] dup_rows_per_dst=[908, 1454, 4304, 1390, 1056, 2214, 372, 5998] combine_rows_per_src=[4774, 5048, 6856, 6364, 4884, 7234, 4982, 7698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47618 payload_rows_remote=40678 dup_rows=17918 dup_rows_remote=16328 dedup_ratio=0.273407 router_tokens_per_expert=[0, 1497, 0, 0, 2, 6, 1742, 436, 0, 2, 0, 526, 0, 1368, 0, 327, 367, 264, 2307, 440, 517, 1442, 455, 1969, 855, 0, 54, 35, 1012, 0, 427, 0, 24, 0, 420, 3145, 0, 0, 183, 493, 0, 460, 43, 66, 2736, 3, 0, 421, 2, 2889, 0, 770, 25, 3, 25, 473, 156, 2, 4, 666, 5, 695, 3, 3006] payload_rows_per_dst=[7128, 6128, 4858, 6932, 6940, 6162, 6746, 2724] dup_rows_per_dst=[4852, 3136, 1232, 3846, 1590, 1296, 1628, 338] combine_rows_per_src=[7128, 6128, 4858, 6932, 6940, 6162, 6746, 2724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49952 payload_rows_remote=43090 dup_rows=15584 dup_rows_remote=13118 dedup_ratio=0.237793 router_tokens_per_expert=[1, 1419, 0, 1, 21, 9, 1133, 652, 2, 1, 0, 102, 1, 262, 0, 31, 604, 172, 661, 1954, 1168, 1309, 265, 2422, 123, 0, 7, 1, 495, 0, 34, 0, 268, 3, 220, 2774, 0, 0, 310, 1949, 0, 2139, 252, 21, 2102, 9, 0, 141, 31, 3329, 2, 260, 97, 1, 94, 55, 193, 2, 0, 2181, 22, 927, 64, 2472] payload_rows_per_dst=[5804, 6394, 5640, 5486, 5090, 6862, 7118, 7558] dup_rows_per_dst=[1990, 2310, 2242, 778, 1046, 2466, 620, 4132] combine_rows_per_src=[5804, 6394, 5640, 5486, 5090, 6862, 7118, 7558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48936 payload_rows_remote=43310 dup_rows=16600 dup_rows_remote=14622 dedup_ratio=0.253296 router_tokens_per_expert=[2, 1562, 1, 10, 9, 12, 825, 599, 7, 0, 6, 116, 62, 241, 28, 247, 454, 124, 1491, 1554, 668, 1477, 1186, 1541, 486, 15, 57, 1, 2170, 5, 88, 14, 321, 21, 189, 2272, 70, 3, 269, 1209, 83, 553, 986, 0, 1602, 220, 5, 59, 15, 2595, 57, 419, 144, 0, 23, 187, 2024, 28, 5, 894, 98, 102, 222, 3035] payload_rows_per_dst=[6418, 5626, 5748, 7450, 6524, 5500, 6194, 5476] dup_rows_per_dst=[2604, 1978, 2070, 4292, 2184, 1516, 686, 1270] combine_rows_per_src=[6418, 5626, 5748, 7450, 6524, 5500, 6194, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=272039936 dispatch_meta_bytes=198736 combine_payload_bytes=272039936 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=272238672 ingress_bytes=492240896 link_bytes=492240896 token_bytes=544278608 weight_movement_bytes=220200960 total_nvlink_bytes=764479568 weight_to_token_ratio=0.404574 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.560878779e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=307206144 dispatch_meta_bytes=240728 combine_payload_bytes=307206144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=307446872 ingress_bytes=417306624 link_bytes=417306624 token_bytes=614653016 weight_movement_bytes=110100480 total_nvlink_bytes=724753496 weight_to_token_ratio=0.179126 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=23808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.257648543e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50564 payload_rows_remote=43600 dup_rows=14972 dup_rows_remote=13916 dedup_ratio=0.228455 router_tokens_per_expert=[0, 1667, 3, 1, 6, 0, 283, 1430, 0, 1, 0, 553, 1, 886, 0, 73, 324, 4, 1428, 706, 312, 1217, 502, 2814, 231, 0, 42, 0, 406, 1, 34, 0, 7, 0, 580, 2103, 0, 0, 387, 986, 0, 1172, 507, 12, 3536, 18, 0, 141, 1, 3607, 1, 169, 282, 3, 48, 563, 50, 0, 0, 1769, 22, 533, 50, 3296] payload_rows_per_dst=[6648, 7004, 4014, 6964, 6214, 7650, 7806, 4264] dup_rows_per_dst=[2988, 3064, 704, 1056, 1912, 3122, 1542, 584] combine_rows_per_src=[6648, 7004, 4014, 6964, 6214, 7650, 7806, 4264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=291579904 dispatch_meta_bytes=228024 combine_payload_bytes=291579904 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=291807928 ingress_bytes=511780864 link_bytes=511780864 token_bytes=583387832 weight_movement_bytes=220200960 total_nvlink_bytes=803588792 weight_to_token_ratio=0.377452 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.900710960e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49438 payload_rows_remote=42736 dup_rows=16098 dup_rows_remote=12730 dedup_ratio=0.245636 router_tokens_per_expert=[0, 1208, 0, 11, 16, 21, 605, 232, 3, 6, 4, 107, 2, 1120, 27, 33, 2231, 226, 1414, 1279, 1214, 1169, 195, 2170, 314, 6, 7, 2, 139, 0, 24, 14, 131, 10, 184, 1826, 0, 0, 1489, 1566, 4, 674, 534, 87, 2168, 29, 0, 2, 179, 2734, 4, 575, 370, 69, 1, 11, 441, 0, 0, 894, 14, 1394, 42, 3537] payload_rows_per_dst=[5368, 6488, 6702, 7160, 6972, 5454, 6508, 4786] dup_rows_per_dst=[1646, 3014, 3368, 926, 3440, 1542, 1378, 784] combine_rows_per_src=[5368, 6488, 6702, 7160, 6972, 5454, 6508, 4786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=308869120 dispatch_meta_bytes=224832 combine_payload_bytes=308869120 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=309093952 ingress_bytes=639170560 link_bytes=639170560 token_bytes=617963072 weight_movement_bytes=330301440 total_nvlink_bytes=948264512 weight_to_token_ratio=0.534500 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.111622729e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=310446080 dispatch_meta_bytes=231728 combine_payload_bytes=310446080 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=310677808 ingress_bytes=530647040 link_bytes=530647040 token_bytes=621123888 weight_movement_bytes=220200960 total_nvlink_bytes=841324848 weight_to_token_ratio=0.354520 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.228824790e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45856 payload_rows_remote=39432 dup_rows=19680 dup_rows_remote=17730 dedup_ratio=0.300293 router_tokens_per_expert=[0, 671, 21, 42, 181, 603, 0, 2461, 0, 2497, 0, 54, 599, 387, 2644, 48, 0, 258, 35, 56, 22, 1, 76, 3601, 32, 0, 0, 163, 43, 89, 607, 3384, 1454, 0, 8, 1638, 172, 407, 8, 2158, 0, 144, 0, 2477, 383, 32, 0, 54, 182, 4, 37, 392, 977, 2486, 17, 0, 6, 0, 163, 517, 61, 88, 326, 2] payload_rows_per_dst=[6424, 7168, 7676, 7664, 5874, 5394, 3480, 2176] dup_rows_per_dst=[1950, 4874, 4644, 4248, 2540, 786, 488, 150] combine_rows_per_src=[6424, 7168, 7676, 7664, 5874, 5394, 3480, 2176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=312524800 dispatch_meta_bytes=230064 combine_payload_bytes=312524800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=312754864 ingress_bytes=422625280 link_bytes=422625280 token_bytes=625279664 weight_movement_bytes=110100480 total_nvlink_bytes=735380144 weight_to_token_ratio=0.176082 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.350148718e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=306331648 dispatch_meta_bytes=221864 combine_payload_bytes=306331648 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=306553512 ingress_bytes=746733568 link_bytes=746733568 token_bytes=612885160 weight_movement_bytes=440401920 total_nvlink_bytes=1053287080 weight_to_token_ratio=0.718572 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=9344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.298692491e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49622 payload_rows_remote=42114 dup_rows=15914 dup_rows_remote=12180 dedup_ratio=0.242828 router_tokens_per_expert=[3, 444, 21, 60, 179, 1017, 7, 3396, 0, 1680, 1, 179, 209, 995, 3403, 5, 1, 8, 5, 877, 244, 2, 42, 3801, 253, 7, 1, 51, 0, 0, 12, 2668, 1341, 0, 95, 2172, 5, 685, 5, 298, 0, 22, 0, 686, 1212, 0, 0, 3, 5, 0, 4, 77, 1155, 3212, 2, 0, 3, 8, 1189, 24, 786, 37, 171, 0] payload_rows_per_dst=[3096, 4618, 7720, 5656, 6586, 7356, 7082, 7508] dup_rows_per_dst=[366, 862, 2240, 328, 2834, 3282, 2268, 3734] combine_rows_per_src=[3096, 4618, 7720, 5656, 6586, 7356, 7082, 7508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49550 payload_rows_remote=43546 dup_rows=15986 dup_rows_remote=15788 dedup_ratio=0.243927 router_tokens_per_expert=[40, 243, 21, 336, 328, 1148, 0, 3933, 0, 2428, 4, 1102, 153, 1127, 3055, 1, 0, 8, 0, 282, 113, 0, 96, 3008, 820, 0, 0, 40, 0, 2, 376, 1433, 1703, 0, 8, 2640, 0, 787, 0, 1046, 0, 14, 0, 140, 1609, 0, 0, 1, 22, 33, 13, 16, 182, 2831, 4, 0, 0, 9, 1064, 387, 27, 68, 67, 0] payload_rows_per_dst=[3606, 6736, 6324, 4594, 7264, 8050, 6004, 6972] dup_rows_per_dst=[626, 2894, 690, 748, 5104, 3344, 198, 2382] combine_rows_per_src=[3606, 6736, 6324, 4594, 7264, 8050, 6004, 6972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=282648576 dispatch_meta_bytes=228648 combine_payload_bytes=282648576 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=282877224 ingress_bytes=502849536 link_bytes=502849536 token_bytes=565525800 weight_movement_bytes=220200960 total_nvlink_bytes=785726760 weight_to_token_ratio=0.389374 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.745380476e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49282 payload_rows_remote=46626 dup_rows=16254 dup_rows_remote=16024 dedup_ratio=0.248016 router_tokens_per_expert=[0, 63, 12, 1, 694, 1663, 0, 2871, 0, 3201, 0, 272, 261, 1019, 3853, 65, 0, 76, 37, 95, 101, 0, 180, 2892, 307, 8, 0, 0, 2, 0, 125, 1593, 381, 0, 136, 2282, 0, 773, 0, 153, 0, 12, 0, 2310, 339, 0, 0, 6, 12, 6, 7, 340, 1187, 3323, 0, 0, 0, 0, 777, 252, 920, 41, 61, 59] payload_rows_per_dst=[4480, 7224, 6014, 6458, 2656, 7256, 7196, 7998] dup_rows_per_dst=[588, 2412, 748, 2176, 230, 3618, 2554, 3928] combine_rows_per_src=[4480, 7224, 6014, 6458, 2656, 7256, 7196, 7998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49628 payload_rows_remote=41890 dup_rows=15908 dup_rows_remote=12714 dedup_ratio=0.242737 router_tokens_per_expert=[1, 1410, 100, 23, 133, 1009, 2, 3656, 2, 1346, 1, 195, 1318, 1001, 3109, 21, 6, 15, 115, 1328, 219, 0, 5, 2231, 194, 5, 1, 55, 0, 3, 60, 1463, 930, 0, 85, 2195, 31, 1101, 0, 1315, 5, 184, 0, 461, 1156, 0, 1, 3, 10, 40, 24, 70, 762, 3587, 2, 0, 4, 2, 872, 277, 34, 191, 390, 9] payload_rows_per_dst=[4352, 6026, 6152, 4134, 6394, 7738, 7622, 7210] dup_rows_per_dst=[1004, 1742, 1686, 490, 3858, 3194, 1368, 2566] combine_rows_per_src=[4352, 6026, 6152, 4134, 6394, 7738, 7622, 7210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=301873152 dispatch_meta_bytes=217176 combine_payload_bytes=301873152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=302090328 ingress_bytes=411973632 link_bytes=411973632 token_bytes=603963480 weight_movement_bytes=110100480 total_nvlink_bytes=714063960 weight_to_token_ratio=0.182297 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.164899041e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47538 payload_rows_remote=41052 dup_rows=17998 dup_rows_remote=15650 dedup_ratio=0.274628 router_tokens_per_expert=[2, 274, 6, 11, 1674, 355, 77, 3973, 2, 1034, 58, 332, 65, 388, 3132, 76, 37, 278, 177, 453, 51, 0, 56, 2026, 88, 116, 0, 300, 2, 18, 2359, 2021, 101, 11, 34, 935, 11, 923, 22, 1965, 0, 538, 0, 967, 824, 10, 1, 0, 552, 52, 1508, 114, 789, 2250, 2, 6, 0, 4, 281, 1073, 40, 47, 267, 0] payload_rows_per_dst=[8028, 6486, 5278, 7198, 5370, 3946, 7166, 4066] dup_rows_per_dst=[4716, 2348, 878, 4480, 764, 734, 3380, 698] combine_rows_per_src=[8028, 6486, 5278, 7198, 5370, 3946, 7166, 4066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=312137728 dispatch_meta_bytes=237336 combine_payload_bytes=312137728 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=312375064 ingress_bytes=532338688 link_bytes=532338688 token_bytes=624512792 weight_movement_bytes=220200960 total_nvlink_bytes=844713752 weight_to_token_ratio=0.352596 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.258245331e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48626 payload_rows_remote=41786 dup_rows=16910 dup_rows_remote=14216 dedup_ratio=0.258026 router_tokens_per_expert=[60, 398, 38, 3, 491, 2011, 0, 3788, 0, 816, 0, 109, 685, 965, 2341, 4, 0, 4, 22, 579, 0, 0, 67, 3659, 28, 1, 3, 0, 0, 0, 559, 2894, 834, 0, 7, 1282, 0, 1461, 0, 1928, 0, 14, 0, 1589, 791, 3, 0, 0, 63, 8, 84, 232, 1918, 1685, 24, 0, 4, 12, 848, 149, 152, 33, 120, 2] payload_rows_per_dst=[8026, 4252, 7430, 6840, 6052, 4216, 6130, 5680] dup_rows_per_dst=[5552, 906, 1232, 2694, 2408, 578, 1898, 1642] combine_rows_per_src=[8026, 4252, 7430, 6840, 6052, 4216, 6130, 5680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=334215168 dispatch_meta_bytes=250600 combine_payload_bytes=334215168 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=334465768 ingress_bytes=554416128 link_bytes=554416128 token_bytes=668680936 weight_movement_bytes=220200960 total_nvlink_bytes=888881896 weight_to_token_ratio=0.329306 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.642208324e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48454 payload_rows_remote=43026 dup_rows=17082 dup_rows_remote=15686 dedup_ratio=0.260651 router_tokens_per_expert=[15, 844, 246, 109, 1053, 331, 0, 3763, 0, 2352, 128, 136, 231, 504, 2607, 143, 0, 11, 131, 1013, 625, 2, 7, 1623, 212, 0, 1, 159, 0, 4, 301, 1884, 1531, 6, 564, 1353, 0, 349, 23, 1328, 0, 376, 20, 1058, 1549, 2, 0, 17, 12, 7, 227, 593, 732, 2780, 0, 0, 0, 106, 329, 185, 292, 75, 819, 0] payload_rows_per_dst=[7936, 5780, 5428, 5826, 5324, 4870, 6716, 6574] dup_rows_per_dst=[4786, 1208, 1396, 2002, 2278, 1174, 1986, 2252] combine_rows_per_src=[7936, 5780, 5428, 5826, 5324, 4870, 6716, 6574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=300267520 dispatch_meta_bytes=218416 combine_payload_bytes=300267520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=300485936 ingress_bytes=410368000 link_bytes=410368000 token_bytes=600753456 weight_movement_bytes=110100480 total_nvlink_bytes=710853936 weight_to_token_ratio=0.183271 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.136974459e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=294260736 dispatch_meta_bytes=226808 combine_payload_bytes=294260736 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=294487544 ingress_bytes=734662656 link_bytes=734662656 token_bytes=588748280 weight_movement_bytes=440401920 total_nvlink_bytes=1029150200 weight_to_token_ratio=0.748031 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=14848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.277699190e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43924 payload_rows_remote=35804 dup_rows=21612 dup_rows_remote=13486 dedup_ratio=0.329773 router_tokens_per_expert=[3322, 761, 0, 0, 3329, 472, 5, 27, 1, 14, 0, 15, 170, 2750, 0, 10, 3, 0, 175, 82, 2, 3691, 17, 17, 889, 2572, 32, 59, 0, 0, 0, 3005, 1, 0, 0, 315, 0, 192, 0, 0, 62, 619, 254, 3764, 10, 3, 368, 0, 79, 18, 1, 0, 265, 101, 26, 720, 28, 115, 0, 28, 532, 3734, 113, 0] payload_rows_per_dst=[8120, 5806, 7498, 4032, 6384, 8190, 2298, 1596] dup_rows_per_dst=[8126, 114, 2338, 796, 642, 9438, 122, 36] combine_rows_per_src=[8120, 5806, 7498, 4032, 6384, 8190, 2298, 1596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=299522048 dispatch_meta_bytes=224008 combine_payload_bytes=299522048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=299746056 ingress_bytes=409622528 link_bytes=409622528 token_bytes=599268104 weight_movement_bytes=110100480 total_nvlink_bytes=709368584 weight_to_token_ratio=0.183725 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.124009475e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=308410368 dispatch_meta_bytes=234848 combine_payload_bytes=308410368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=308645216 ingress_bytes=418510848 link_bytes=418510848 token_bytes=617055584 weight_movement_bytes=110100480 total_nvlink_bytes=727156064 weight_to_token_ratio=0.178429 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.278591979e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47578 payload_rows_remote=39512 dup_rows=17958 dup_rows_remote=12926 dedup_ratio=0.274017 router_tokens_per_expert=[508, 481, 1, 3, 955, 83, 6, 1, 15, 7, 0, 5, 9, 3617, 0, 0, 2, 5, 111, 600, 1, 3733, 263, 0, 1542, 3675, 1, 16, 0, 2, 0, 3616, 5, 2, 0, 282, 0, 84, 1, 1, 2, 871, 23, 3159, 6, 7, 1480, 0, 1, 4, 6, 6, 150, 28, 198, 1479, 4, 280, 0, 18, 1336, 3838, 239, 0] payload_rows_per_dst=[1372, 7462, 7722, 3598, 7808, 4042, 7508, 8066] dup_rows_per_dst=[20, 860, 1708, 350, 6698, 736, 2554, 5032] combine_rows_per_src=[1372, 7462, 7722, 3598, 7808, 4042, 7508, 8066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=256643072 dispatch_meta_bytes=197160 combine_payload_bytes=256643072 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=256840232 ingress_bytes=586944512 link_bytes=586944512 token_bytes=513483304 weight_movement_bytes=330301440 total_nvlink_bytes=843784744 weight_to_token_ratio=0.643256 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.020793042e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49662 payload_rows_remote=41618 dup_rows=15874 dup_rows_remote=12358 dedup_ratio=0.242218 router_tokens_per_expert=[1523, 483, 1, 1, 1153, 667, 8, 0, 0, 0, 0, 0, 0, 3719, 0, 1, 1, 0, 272, 919, 1, 1897, 3, 1, 2689, 3758, 159, 7, 0, 0, 0, 3337, 0, 0, 0, 130, 0, 752, 2, 0, 11, 63, 400, 3816, 0, 0, 593, 0, 4, 3, 12, 4, 399, 25, 96, 1421, 0, 78, 0, 74, 764, 3475, 45, 1] payload_rows_per_dst=[4100, 7798, 5254, 7884, 7036, 2110, 8044, 7436] dup_rows_per_dst=[526, 2688, 934, 5342, 1406, 24, 3516, 1438] combine_rows_per_src=[4100, 7798, 5254, 7884, 7036, 2110, 8044, 7436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51094 payload_rows_remote=44370 dup_rows=14442 dup_rows_remote=13384 dedup_ratio=0.220367 router_tokens_per_expert=[2138, 2432, 0, 0, 1487, 59, 0, 4, 0, 0, 0, 0, 24, 3243, 0, 64, 0, 0, 151, 1399, 0, 1889, 22, 57, 1360, 1892, 38, 1, 0, 1, 0, 3110, 9, 0, 0, 690, 0, 82, 0, 0, 3, 487, 1326, 3596, 1, 0, 139, 0, 0, 0, 1, 0, 130, 9, 68, 1783, 2, 770, 0, 233, 128, 3920, 20, 0] payload_rows_per_dst=[6438, 7444, 5610, 5388, 6724, 3672, 7916, 7902] dup_rows_per_dst=[1526, 3494, 1426, 1196, 1058, 240, 3258, 2244] combine_rows_per_src=[6438, 7444, 5610, 5388, 6724, 3672, 7916, 7902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=283222016 dispatch_meta_bytes=209752 combine_payload_bytes=283222016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=283431768 ingress_bytes=503422976 link_bytes=503422976 token_bytes=566653784 weight_movement_bytes=220200960 total_nvlink_bytes=786854744 weight_to_token_ratio=0.388599 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.755353541e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51344 payload_rows_remote=47292 dup_rows=14192 dup_rows_remote=13724 dedup_ratio=0.216553 router_tokens_per_expert=[2957, 517, 0, 0, 1414, 142, 1, 9, 27, 0, 0, 33, 15, 2029, 0, 1, 0, 1, 254, 688, 3, 3251, 65, 56, 1302, 2762, 172, 25, 0, 0, 2, 2774, 2, 1, 0, 242, 0, 1409, 0, 0, 5, 302, 355, 3671, 0, 0, 1598, 0, 1, 38, 6, 0, 34, 663, 247, 588, 0, 251, 3, 41, 843, 3823, 145, 0] payload_rows_per_dst=[3670, 7338, 7164, 6390, 6822, 4052, 7886, 8022] dup_rows_per_dst=[496, 2786, 1472, 2136, 2034, 468, 2610, 2190] combine_rows_per_src=[3670, 7338, 7164, 6390, 6822, 4052, 7886, 8022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=298317824 dispatch_meta_bytes=215904 combine_payload_bytes=298317824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298533728 ingress_bytes=408418304 link_bytes=408418304 token_bytes=596851552 weight_movement_bytes=110100480 total_nvlink_bytes=706952032 weight_to_token_ratio=0.184469 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.103066039e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45314 payload_rows_remote=39440 dup_rows=20222 dup_rows_remote=19944 dedup_ratio=0.308563 router_tokens_per_expert=[1054, 2674, 0, 27, 1841, 207, 91, 15, 1, 0, 2, 0, 231, 2754, 0, 88, 10, 2, 35, 1209, 1, 3299, 6, 59, 1783, 1453, 8, 281, 9, 31, 1, 2480, 0, 109, 0, 141, 1, 221, 8, 6, 1306, 1160, 8, 2116, 0, 9, 1334, 90, 9, 57, 0, 2, 1, 63, 125, 1678, 1, 162, 26, 172, 680, 3625, 5, 1] payload_rows_per_dst=[7460, 5874, 7380, 5600, 5402, 7958, 3678, 1962] dup_rows_per_dst=[4358, 278, 1862, 1532, 530, 11338, 192, 132] combine_rows_per_src=[7460, 5874, 7380, 5600, 5402, 7958, 3678, 1962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=318044160 dispatch_meta_bytes=231016 combine_payload_bytes=318044160 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=318275176 ingress_bytes=538245120 link_bytes=538245120 token_bytes=636319336 weight_movement_bytes=220200960 total_nvlink_bytes=856520296 weight_to_token_ratio=0.346054 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.033088 gemm_active_groups=11 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.360967898e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51928 payload_rows_remote=44388 dup_rows=13608 dup_rows_remote=8734 dedup_ratio=0.207642 router_tokens_per_expert=[1564, 673, 0, 0, 1961, 183, 0, 1, 0, 0, 0, 2, 11, 2063, 0, 3, 0, 0, 278, 1893, 0, 1971, 34, 0, 2828, 3193, 167, 19, 0, 0, 0, 3709, 0, 0, 0, 132, 0, 330, 0, 0, 5, 331, 388, 3484, 0, 0, 1571, 0, 2, 0, 1, 1, 91, 165, 16, 937, 0, 206, 0, 216, 529, 3741, 69, 0] payload_rows_per_dst=[5024, 5774, 6588, 7540, 7508, 4312, 7300, 7882] dup_rows_per_dst=[612, 1512, 1764, 4874, 834, 278, 2094, 1640] combine_rows_per_src=[5024, 5774, 6588, 7540, 7508, 4312, 7300, 7882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=338989056 dispatch_meta_bytes=244064 combine_payload_bytes=338989056 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=339233120 ingress_bytes=559190016 link_bytes=559190016 token_bytes=678222176 weight_movement_bytes=220200960 total_nvlink_bytes=898423136 weight_to_token_ratio=0.324674 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.725234088e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47434 payload_rows_remote=41728 dup_rows=18102 dup_rows_remote=16992 dedup_ratio=0.276215 router_tokens_per_expert=[2206, 2020, 0, 0, 1882, 1480, 1, 1, 3, 4, 3, 2, 75, 2319, 0, 2, 11, 5, 44, 1816, 0, 1081, 421, 30, 2609, 1533, 80, 187, 30, 0, 0, 2101, 16, 27, 3, 74, 0, 100, 0, 4, 42, 1136, 71, 2829, 1, 0, 935, 0, 0, 34, 1, 4, 8, 155, 2, 804, 7, 1237, 0, 56, 1473, 3635, 159, 9] payload_rows_per_dst=[7420, 5318, 5706, 6548, 4432, 5466, 5656, 6888] dup_rows_per_dst=[6268, 990, 1110, 2330, 218, 3212, 904, 3070] combine_rows_per_src=[7420, 5318, 5706, 6548, 4432, 5466, 5656, 6888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46934 payload_rows_remote=38788 dup_rows=18602 dup_rows_remote=14034 dedup_ratio=0.283844 router_tokens_per_expert=[0, 304, 77, 1269, 1, 2, 0, 681, 0, 459, 5, 1, 568, 0, 1092, 0, 126, 0, 392, 1495, 0, 1246, 66, 0, 1859, 0, 4023, 816, 8, 1176, 1457, 2410, 2996, 17, 64, 1, 0, 1, 30, 0, 0, 7, 0, 219, 5, 2459, 82, 7, 801, 0, 3, 3, 624, 0, 1, 3822, 1577, 0, 2, 260, 222, 1, 30, 1] payload_rows_per_dst=[8146, 5302, 8082, 7616, 6110, 5234, 2642, 3802] dup_rows_per_dst=[4568, 1242, 6212, 5542, 108, 324, 222, 384] combine_rows_per_src=[8146, 5302, 8082, 7616, 6110, 5234, 2642, 3802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=282705920 dispatch_meta_bytes=237536 combine_payload_bytes=282705920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282943456 ingress_bytes=392806400 link_bytes=392806400 token_bytes=565649376 weight_movement_bytes=110100480 total_nvlink_bytes=675749856 weight_to_token_ratio=0.194644 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=22528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.831549352e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=318173184 dispatch_meta_bytes=212488 combine_payload_bytes=318173184 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=318385672 ingress_bytes=758575104 link_bytes=758575104 token_bytes=636558856 weight_movement_bytes=440401920 total_nvlink_bytes=1076960776 weight_to_token_ratio=0.691848 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.319286870e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48358 payload_rows_remote=42008 dup_rows=17178 dup_rows_remote=14580 dedup_ratio=0.262115 router_tokens_per_expert=[1, 425, 0, 1351, 1, 0, 3, 260, 0, 2090, 1, 0, 1067, 41, 1583, 0, 0, 1, 861, 1284, 1, 2059, 1, 0, 1660, 1, 3760, 72, 747, 435, 1639, 2166, 1448, 194, 3, 4, 2, 0, 94, 0, 5, 12, 0, 1349, 25, 1069, 589, 0, 123, 0, 9, 0, 230, 12, 6, 3776, 1908, 0, 0, 270, 97, 27, 2, 4] payload_rows_per_dst=[3708, 5404, 5968, 4790, 7878, 6418, 7842, 6350] dup_rows_per_dst=[374, 994, 2446, 998, 3132, 2846, 3790, 2598] combine_rows_per_src=[3708, 5404, 5968, 4790, 7878, 6418, 7842, 6350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=299106304 dispatch_meta_bytes=234880 combine_payload_bytes=299106304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=299341184 ingress_bytes=409206784 link_bytes=409206784 token_bytes=598447488 weight_movement_bytes=110100480 total_nvlink_bytes=708547968 weight_to_token_ratio=0.183977 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.116779003e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278032384 dispatch_meta_bytes=211288 combine_payload_bytes=278032384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278243672 ingress_bytes=388132864 link_bytes=388132864 token_bytes=556276056 weight_movement_bytes=110100480 total_nvlink_bytes=666376536 weight_to_token_ratio=0.197924 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750268874e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51752 payload_rows_remote=43592 dup_rows=13784 dup_rows_remote=13534 dedup_ratio=0.210327 router_tokens_per_expert=[230, 432, 0, 2340, 4, 1, 29, 996, 1, 2186, 0, 5, 787, 91, 3129, 0, 22, 0, 68, 1925, 0, 825, 0, 0, 1589, 0, 3856, 476, 191, 1456, 330, 1647, 937, 1, 1, 27, 0, 0, 54, 0, 0, 2, 0, 326, 51, 1597, 448, 0, 7, 0, 17, 0, 12, 0, 19, 4079, 2348, 0, 6, 77, 76, 12, 54, 1] payload_rows_per_dst=[6052, 5392, 4886, 5676, 7830, 7584, 8160, 6172] dup_rows_per_dst=[2012, 748, 794, 2266, 1922, 3522, 250, 2270] combine_rows_per_src=[6052, 5392, 4886, 5676, 7830, 7584, 8160, 6172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51758 payload_rows_remote=43568 dup_rows=13778 dup_rows_remote=12828 dedup_ratio=0.210236 router_tokens_per_expert=[13, 55, 0, 2614, 0, 1, 0, 2853, 0, 1187, 0, 0, 548, 103, 2968, 0, 3, 0, 192, 1571, 0, 642, 13, 0, 734, 0, 4092, 7, 799, 531, 252, 1791, 229, 74, 105, 0, 0, 0, 70, 0, 0, 18, 0, 3587, 9, 886, 363, 0, 158, 0, 298, 0, 326, 0, 2, 3948, 1103, 0, 3, 460, 101, 43, 12, 4] payload_rows_per_dst=[7592, 6360, 4124, 3948, 8190, 7798, 8038, 5708] dup_rows_per_dst=[3480, 1752, 718, 698, 950, 3428, 1426, 1326] combine_rows_per_src=[7592, 6360, 4124, 3948, 8190, 7798, 8038, 5708] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=301113344 dispatch_meta_bytes=226352 combine_payload_bytes=301113344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=301339696 ingress_bytes=411213824 link_bytes=411213824 token_bytes=602453040 weight_movement_bytes=110100480 total_nvlink_bytes=712553520 weight_to_token_ratio=0.182754 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.151684730e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50816 payload_rows_remote=43594 dup_rows=14720 dup_rows_remote=10546 dedup_ratio=0.224609 router_tokens_per_expert=[142, 451, 2, 1548, 12, 0, 0, 944, 2, 2954, 8, 0, 436, 64, 2241, 0, 0, 1, 76, 1345, 6, 1336, 61, 2, 1129, 5, 3962, 201, 31, 789, 758, 2137, 996, 24, 37, 11, 7, 14, 133, 0, 0, 6, 0, 1896, 5, 1368, 181, 1, 484, 1, 7, 8, 247, 1, 1, 4029, 2299, 0, 39, 197, 82, 5, 2, 44] payload_rows_per_dst=[4922, 6354, 4660, 4862, 8014, 7222, 8110, 6672] dup_rows_per_dst=[1276, 574, 994, 964, 2354, 4174, 1446, 2938] combine_rows_per_src=[4922, 6354, 4660, 4862, 8014, 7222, 8110, 6672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=312467456 dispatch_meta_bytes=228504 combine_payload_bytes=312467456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=312695960 ingress_bytes=532668416 link_bytes=532668416 token_bytes=625163416 weight_movement_bytes=220200960 total_nvlink_bytes=845364376 weight_to_token_ratio=0.352229 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.263979843e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46062 payload_rows_remote=39554 dup_rows=19474 dup_rows_remote=17818 dedup_ratio=0.297150 router_tokens_per_expert=[45, 331, 1, 1861, 21, 3, 32, 899, 1, 2940, 2, 1, 73, 12, 1048, 5, 11, 2, 31, 2804, 4, 919, 49, 72, 15, 6, 3592, 1458, 479, 581, 494, 2148, 2374, 12, 5, 67, 22, 17, 19, 19, 0, 0, 4, 769, 27, 167, 352, 20, 986, 6, 399, 1, 289, 3, 196, 3645, 2312, 0, 12, 944, 10, 30, 43, 78] payload_rows_per_dst=[8040, 6508, 8024, 6906, 4990, 2612, 3222, 5760] dup_rows_per_dst=[5530, 1656, 7050, 3456, 80, 66, 538, 1098] combine_rows_per_src=[8040, 6508, 8024, 6906, 4990, 2612, 3222, 5760] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=312295424 dispatch_meta_bytes=225584 combine_payload_bytes=312295424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=312521008 ingress_bytes=422395904 link_bytes=422395904 token_bytes=624816432 weight_movement_bytes=110100480 total_nvlink_bytes=734916912 weight_to_token_ratio=0.176213 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.346159492e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=312481792 dispatch_meta_bytes=216560 combine_payload_bytes=312481792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=312698352 ingress_bytes=422582272 link_bytes=422582272 token_bytes=625180144 weight_movement_bytes=110100480 total_nvlink_bytes=735280624 weight_to_token_ratio=0.176110 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.349400738e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48648 payload_rows_remote=43894 dup_rows=16888 dup_rows_remote=14234 dedup_ratio=0.257690 router_tokens_per_expert=[227, 37, 1, 1086, 1, 0, 0, 1394, 1, 2020, 0, 5, 805, 13, 2616, 0, 53, 0, 331, 1258, 0, 1020, 1, 0, 473, 0, 4039, 140, 142, 1230, 1167, 1793, 1313, 49, 2, 0, 0, 0, 45, 0, 0, 36, 0, 1052, 8, 2291, 470, 13, 37, 0, 31, 0, 668, 2, 0, 4024, 2299, 0, 6, 165, 195, 85, 65, 59] payload_rows_per_dst=[4620, 7258, 4216, 4754, 7294, 6024, 8088, 6394] dup_rows_per_dst=[872, 3662, 1110, 2654, 2498, 1716, 1436, 2940] combine_rows_per_src=[4620, 7258, 4216, 4754, 7294, 6024, 8088, 6394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47018 payload_rows_remote=40546 dup_rows=18518 dup_rows_remote=16362 dedup_ratio=0.282562 router_tokens_per_expert=[10, 365, 1049, 132, 0, 2648, 0, 2, 1841, 1331, 1, 1, 0, 178, 0, 1458, 3, 112, 0, 1, 5, 39, 3281, 20, 2, 0, 496, 0, 5, 2561, 1619, 2, 0, 0, 0, 0, 49, 1, 71, 2234, 43, 987, 0, 343, 1471, 1, 241, 2, 1, 3, 0, 14, 3529, 2500, 69, 813, 44, 30, 2, 259, 427, 2368, 103, 1] payload_rows_per_dst=[6472, 7686, 6534, 3874, 7948, 7220, 1786, 5498] dup_rows_per_dst=[2156, 7056, 172, 374, 3820, 3956, 14, 970] combine_rows_per_src=[6472, 7686, 6534, 3874, 7948, 7220, 1786, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283523072 dispatch_meta_bytes=229488 combine_payload_bytes=283523072 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=283752560 ingress_bytes=503724032 link_bytes=503724032 token_bytes=567275632 weight_movement_bytes=220200960 total_nvlink_bytes=787476592 weight_to_token_ratio=0.388173 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.760589400e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48146 payload_rows_remote=43206 dup_rows=17390 dup_rows_remote=16078 dedup_ratio=0.265350 router_tokens_per_expert=[52, 610, 1, 815, 4, 14, 2, 266, 3, 2371, 0, 4, 178, 24, 3570, 15, 3, 0, 107, 1268, 0, 1499, 24, 14, 163, 6, 3373, 466, 25, 1211, 297, 2582, 1920, 52, 0, 9, 10, 0, 101, 1, 1, 17, 5, 1250, 12, 1735, 245, 19, 412, 1, 53, 0, 866, 22, 3, 4017, 1809, 2, 5, 699, 116, 282, 41, 96] payload_rows_per_dst=[3586, 7820, 4940, 7346, 4058, 5358, 7880, 7158] dup_rows_per_dst=[758, 4510, 1312, 3060, 128, 1210, 2446, 3966] combine_rows_per_src=[3586, 7820, 4940, 7346, 4058, 5358, 7880, 7158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=314632192 dispatch_meta_bytes=232512 combine_payload_bytes=314632192 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=314864704 ingress_bytes=865134592 link_bytes=865134592 token_bytes=629496896 weight_movement_bytes=550502400 total_nvlink_bytes=1179999296 weight_to_token_ratio=0.874512 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.504611346e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44794 payload_rows_remote=37854 dup_rows=20742 dup_rows_remote=17392 dedup_ratio=0.316498 router_tokens_per_expert=[1, 1169, 2421, 3, 0, 1541, 8, 0, 1664, 1064, 5, 2, 0, 3, 2, 2323, 0, 0, 0, 1, 114, 143, 2905, 4, 21, 3, 1556, 0, 0, 3184, 770, 106, 0, 0, 29, 19, 1, 0, 0, 1182, 49, 501, 1, 602, 512, 5, 24, 2, 199, 7, 28, 203, 3769, 2541, 4, 1255, 157, 21, 25, 87, 316, 2178, 37, 1] payload_rows_per_dst=[6238, 4660, 5802, 7734, 2442, 2880, 8098, 6940] dup_rows_per_dst=[966, 820, 260, 3818, 20, 512, 10996, 3350] combine_rows_per_src=[6238, 4660, 5802, 7734, 2442, 2880, 8098, 6940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46016 payload_rows_remote=37832 dup_rows=19520 dup_rows_remote=10130 dedup_ratio=0.297852 router_tokens_per_expert=[0, 1121, 95, 49, 1, 542, 1, 0, 1466, 1851, 0, 0, 0, 140, 0, 1842, 4, 0, 0, 13, 221, 71, 3619, 23, 2, 0, 1347, 20, 0, 1605, 250, 0, 0, 0, 314, 17, 27, 0, 13, 1100, 19, 2692, 1, 1980, 454, 2, 23, 0, 135, 85, 0, 33, 3956, 2461, 0, 1575, 77, 30, 26, 459, 775, 2231, 0, 0] payload_rows_per_dst=[2468, 5484, 7330, 5424, 2772, 7180, 8184, 7174] dup_rows_per_dst=[66, 1430, 572, 1024, 170, 3162, 9390, 3706] combine_rows_per_src=[2468, 5484, 7330, 5424, 2772, 7180, 8184, 7174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=290633728 dispatch_meta_bytes=227632 combine_payload_bytes=290633728 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=290861360 ingress_bytes=510834688 link_bytes=510834688 token_bytes=581495088 weight_movement_bytes=220200960 total_nvlink_bytes=801696048 weight_to_token_ratio=0.378681 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.884255403e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=309700608 dispatch_meta_bytes=237136 combine_payload_bytes=309700608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=309937744 ingress_bytes=419801088 link_bytes=419801088 token_bytes=619638352 weight_movement_bytes=110100480 total_nvlink_bytes=729738832 weight_to_token_ratio=0.177685 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.301031374e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50238 payload_rows_remote=42276 dup_rows=15298 dup_rows_remote=10790 dedup_ratio=0.233429 router_tokens_per_expert=[0, 1493, 838, 391, 0, 1062, 0, 0, 395, 1483, 2, 0, 0, 6, 0, 3437, 31, 1, 0, 3, 4, 3, 4047, 5, 0, 0, 588, 0, 9, 1474, 492, 12, 0, 0, 14, 8, 4, 0, 17, 2766, 0, 406, 0, 568, 1915, 16, 0, 0, 0, 39, 0, 305, 3426, 2064, 57, 2099, 3, 3, 0, 833, 418, 1977, 54, 0] payload_rows_per_dst=[6008, 5276, 8098, 4480, 7962, 6904, 4588, 6922] dup_rows_per_dst=[1560, 1112, 90, 670, 4508, 3034, 412, 3912] combine_rows_per_src=[6008, 5276, 8098, 4480, 7962, 6904, 4588, 6922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48350 payload_rows_remote=44094 dup_rows=17186 dup_rows_remote=16372 dedup_ratio=0.262238 router_tokens_per_expert=[2, 344, 2223, 72, 7, 2109, 12, 4, 2725, 1239, 0, 4, 0, 47, 0, 2081, 5, 14, 1, 0, 30, 230, 3501, 9, 1, 0, 76, 0, 2, 2048, 860, 0, 0, 1, 23, 1, 2, 15, 77, 1848, 5, 98, 0, 853, 1386, 0, 98, 2, 40, 43, 0, 133, 3657, 1503, 2, 2277, 639, 5, 142, 545, 511, 1203, 12, 1] payload_rows_per_dst=[6618, 6518, 7144, 5110, 5270, 4256, 6918, 6516] dup_rows_per_dst=[2658, 1512, 436, 864, 1484, 814, 5656, 3762] combine_rows_per_src=[6618, 6518, 7144, 5110, 5270, 4256, 6918, 6516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=271337472 dispatch_meta_bytes=220984 combine_payload_bytes=271337472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=271558456 ingress_bytes=381437952 link_bytes=381437952 token_bytes=542895928 weight_movement_bytes=110100480 total_nvlink_bytes=652996408 weight_to_token_ratio=0.202802 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.633833343e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=271179776 dispatch_meta_bytes=191848 combine_payload_bytes=271179776 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=271371624 ingress_bytes=601481216 link_bytes=601481216 token_bytes=542551400 weight_movement_bytes=330301440 total_nvlink_bytes=872852840 weight_to_token_ratio=0.608793 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.046074761e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49434 payload_rows_remote=44168 dup_rows=16102 dup_rows_remote=15318 dedup_ratio=0.245697 router_tokens_per_expert=[0, 1334, 1802, 411, 0, 3086, 3, 0, 442, 1739, 0, 33, 14, 4, 1, 472, 2, 0, 14, 0, 37, 223, 3222, 61, 4, 57, 10, 1, 48, 3705, 1866, 2, 7, 1, 393, 14, 0, 173, 688, 298, 23, 67, 10, 1708, 688, 3, 35, 27, 5, 5, 276, 1717, 1926, 1325, 47, 572, 925, 43, 61, 19, 1952, 1107, 27, 33] payload_rows_per_dst=[7658, 5266, 6804, 7432, 5694, 5868, 4504, 6208] dup_rows_per_dst=[5614, 784, 314, 3314, 1306, 1904, 740, 2126] combine_rows_per_src=[7658, 5266, 6804, 7432, 5694, 5868, 4504, 6208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=303034368 dispatch_meta_bytes=212264 combine_payload_bytes=303034368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=303246632 ingress_bytes=413134848 link_bytes=413134848 token_bytes=606281000 weight_movement_bytes=110100480 total_nvlink_bytes=716381480 weight_to_token_ratio=0.181600 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.185094497e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=316065792 dispatch_meta_bytes=241864 combine_payload_bytes=316065792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316307656 ingress_bytes=426166272 link_bytes=426166272 token_bytes=632373448 weight_movement_bytes=110100480 total_nvlink_bytes=742473928 weight_to_token_ratio=0.174107 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.411732393e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48806 payload_rows_remote=41590 dup_rows=16730 dup_rows_remote=13394 dedup_ratio=0.255280 router_tokens_per_expert=[0, 2210, 298, 86, 0, 725, 0, 0, 1717, 932, 0, 0, 0, 432, 0, 1898, 0, 0, 0, 3, 100, 160, 4039, 144, 6, 0, 1625, 14, 0, 2685, 946, 0, 0, 0, 4, 49, 23, 0, 73, 1857, 0, 151, 8, 1062, 1371, 3, 230, 0, 31, 3, 0, 250, 3986, 2681, 0, 969, 14, 39, 1, 1277, 293, 366, 7, 0] payload_rows_per_dst=[5626, 6910, 8084, 7216, 8066, 7050, 2350, 3504] dup_rows_per_dst=[1012, 3048, 808, 3336, 3918, 3962, 156, 490] combine_rows_per_src=[5626, 6910, 8084, 7216, 8066, 7050, 2350, 3504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44578 payload_rows_remote=41660 dup_rows=20958 dup_rows_remote=20728 dedup_ratio=0.319794 router_tokens_per_expert=[0, 20, 0, 213, 12, 406, 1, 3, 0, 40, 111, 7, 13, 1341, 20, 9, 0, 0, 263, 3, 143, 44, 48, 33, 1120, 126, 1087, 919, 0, 0, 4, 547, 3519, 13, 344, 1629, 799, 2, 0, 11, 521, 17, 0, 2507, 4046, 0, 16, 1176, 9, 59, 23, 0, 1060, 12, 0, 1, 2537, 0, 0, 3616, 0, 881, 769, 2668] payload_rows_per_dst=[2918, 8126, 7910, 7280, 7962, 6536, 2308, 1538] dup_rows_per_dst=[230, 3048, 5464, 3824, 6434, 1938, 20, 0] combine_rows_per_src=[2918, 8126, 7910, 7280, 7962, 6536, 2308, 1538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=316596224 dispatch_meta_bytes=237944 combine_payload_bytes=316596224 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=316834168 ingress_bytes=536797184 link_bytes=536797184 token_bytes=633430392 weight_movement_bytes=220200960 total_nvlink_bytes=853631352 weight_to_token_ratio=0.347632 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.335785909e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46614 payload_rows_remote=38524 dup_rows=18922 dup_rows_remote=13924 dedup_ratio=0.288727 router_tokens_per_expert=[29, 476, 2374, 19, 0, 2290, 9, 11, 1317, 453, 76, 4, 1, 0, 1, 2135, 11, 37, 0, 0, 93, 1976, 3749, 678, 4, 3, 885, 6, 35, 2863, 1623, 14, 11, 0, 0, 43, 22, 0, 10, 1813, 12, 29, 1, 1530, 186, 10, 23, 18, 32, 3, 1, 330, 3179, 1608, 76, 430, 186, 5, 28, 752, 573, 472, 213, 0] payload_rows_per_dst=[6902, 6204, 8090, 7216, 7480, 5386, 1640, 3696] dup_rows_per_dst=[3514, 1770, 4998, 3650, 2676, 1448, 104, 762] combine_rows_per_src=[6902, 6204, 8090, 7216, 7480, 5386, 1640, 3696] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45028 payload_rows_remote=37280 dup_rows=20508 dup_rows_remote=13008 dedup_ratio=0.312927 router_tokens_per_expert=[1, 116, 0, 2482, 36, 348, 0, 0, 0, 1, 176, 4, 0, 396, 95, 4, 12, 0, 26, 2, 40, 4, 0, 131, 360, 208, 2229, 3258, 0, 0, 0, 1142, 3500, 21, 47, 699, 825, 7, 0, 10, 7, 72, 4, 54, 4044, 4, 11, 919, 21, 50, 301, 0, 1486, 2, 89, 1, 1990, 1, 0, 1899, 0, 2596, 1627, 1410] payload_rows_per_dst=[5342, 1276, 4014, 7894, 2946, 8132, 7676, 7748] dup_rows_per_dst=[624, 76, 214, 6500, 272, 2098, 3224, 7500] combine_rows_per_src=[5342, 1276, 4014, 7894, 2946, 8132, 7676, 7748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45846 payload_rows_remote=37842 dup_rows=19690 dup_rows_remote=15662 dedup_ratio=0.300446 router_tokens_per_expert=[0, 137, 0, 978, 166, 75, 1, 0, 10, 1, 39, 0, 1, 771, 221, 34, 0, 0, 17, 0, 19, 0, 0, 187, 401, 4, 388, 3400, 0, 0, 3, 2042, 3869, 0, 1, 44, 592, 3, 0, 101, 0, 101, 2, 490, 4086, 0, 158, 574, 82, 35, 901, 0, 1076, 7, 39, 7, 2090, 0, 0, 3159, 0, 3627, 1016, 1813] payload_rows_per_dst=[5850, 2056, 6402, 5802, 1450, 8182, 8004, 8100] dup_rows_per_dst=[1802, 98, 362, 1736, 32, 2640, 4028, 8992] combine_rows_per_src=[5850, 2056, 6402, 5802, 1450, 8182, 8004, 8100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=298117120 dispatch_meta_bytes=219936 combine_payload_bytes=298117120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=298337056 ingress_bytes=518318080 link_bytes=518318080 token_bytes=596454176 weight_movement_bytes=220200960 total_nvlink_bytes=816655136 weight_to_token_ratio=0.369183 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.014403898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=298618880 dispatch_meta_bytes=249552 combine_payload_bytes=298618880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298868432 ingress_bytes=408719360 link_bytes=408719360 token_bytes=597487312 weight_movement_bytes=110100480 total_nvlink_bytes=707587792 weight_to_token_ratio=0.184272 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.108301898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276140032 dispatch_meta_bytes=209792 combine_payload_bytes=276140032 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=276349824 ingress_bytes=496340992 link_bytes=496340992 token_bytes=552489856 weight_movement_bytes=220200960 total_nvlink_bytes=772690816 weight_to_token_ratio=0.398561 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=28160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.632186192e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=201152 combine_payload_bytes=267223040 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=267424192 ingress_bytes=817725440 link_bytes=817725440 token_bytes=534647232 weight_movement_bytes=550502400 total_nvlink_bytes=1085149632 weight_to_token_ratio=1.029655 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=8960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.422159033e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49042 payload_rows_remote=40982 dup_rows=16494 dup_rows_remote=12912 dedup_ratio=0.251678 router_tokens_per_expert=[17, 70, 0, 1846, 43, 200, 1, 2, 1, 13, 665, 36, 6, 1428, 71, 8, 7, 0, 5, 31, 25, 2, 2, 420, 2059, 54, 447, 3710, 0, 1, 3, 344, 3498, 0, 33, 1501, 158, 80, 0, 4, 164, 9, 0, 526, 4014, 0, 79, 1029, 31, 84, 586, 0, 1655, 1, 25, 1, 1890, 1, 1, 3112, 12, 336, 1171, 1250] payload_rows_per_dst=[7878, 3912, 6498, 5078, 3430, 8060, 7758, 6428] dup_rows_per_dst=[3900, 544, 710, 738, 122, 3582, 4004, 2894] combine_rows_per_src=[7878, 3912, 6498, 5078, 3430, 8060, 7758, 6428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48192 payload_rows_remote=43780 dup_rows=17344 dup_rows_remote=16114 dedup_ratio=0.264648 router_tokens_per_expert=[0, 98, 0, 881, 1705, 49, 0, 0, 0, 111, 529, 6, 32, 94, 505, 32, 0, 0, 53, 1, 8, 1, 0, 50, 1451, 156, 987, 3360, 0, 0, 0, 598, 4077, 196, 28, 898, 859, 14, 0, 0, 3, 793, 14, 13, 4096, 5, 15, 900, 6, 2, 120, 0, 926, 1, 71, 0, 2734, 0, 0, 2724, 0, 800, 2143, 623] payload_rows_per_dst=[7560, 7278, 5570, 5088, 4412, 4324, 6890, 7070] dup_rows_per_dst=[4626, 2402, 104, 1296, 1230, 1206, 1864, 4616] combine_rows_per_src=[7560, 7278, 5570, 5088, 4412, 4324, 6890, 7070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=271251456 dispatch_meta_bytes=214016 combine_payload_bytes=271251456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=271465472 ingress_bytes=381351936 link_bytes=381351936 token_bytes=542716928 weight_movement_bytes=110100480 total_nvlink_bytes=652817408 weight_to_token_ratio=0.202869 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.632337383e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5751019672 prefetch_bytes=990904320 reduce_bytes=660602880 weight_movement_bytes=1651507200 egress_bytes=2876637336 ingress_bytes=4525889536 link_bytes=4525889536 total_nvlink_bytes=7402526872 weight_to_token_ratio=0.287168 prefetch_share_of_ingress=0.218941 dedup_ratio=0.291318 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=6 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.871266284e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49118 payload_rows_remote=40936 dup_rows=16418 dup_rows_remote=11440 dedup_ratio=0.250519 router_tokens_per_expert=[122, 1, 1, 165, 311, 35, 0, 3, 2, 138, 176, 6, 21, 2060, 5, 104, 2, 16, 559, 0, 67, 283, 143, 11, 329, 114, 253, 943, 37, 4, 6, 2923, 3558, 681, 37, 502, 1252, 29, 1, 79, 6, 13, 71, 1147, 4068, 8, 194, 1695, 196, 13, 104, 11, 3657, 0, 515, 101, 214, 14, 0, 3258, 28, 301, 1246, 929] payload_rows_per_dst=[2978, 8182, 7352, 6902, 8086, 5292, 7752, 2574] dup_rows_per_dst=[184, 4978, 1754, 2288, 4794, 976, 1442, 2] combine_rows_per_src=[2978, 8182, 7352, 6902, 8086, 5292, 7752, 2574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5599242504 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2800697608 ingress_bytes=4780353536 link_bytes=4780353536 total_nvlink_bytes=7581051144 weight_to_token_ratio=0.353942 prefetch_share_of_ingress=0.207287 dedup_ratio=0.294965 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.313821032e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=293758976 dispatch_meta_bytes=215576 combine_payload_bytes=293758976 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=293974552 ingress_bytes=513959936 link_bytes=513959936 token_bytes=587733528 weight_movement_bytes=220200960 total_nvlink_bytes=807934488 weight_to_token_ratio=0.374661 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.938608606e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=313815040 dispatch_meta_bytes=239576 combine_payload_bytes=313815040 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=314054616 ingress_bytes=534016000 link_bytes=534016000 token_bytes=627869656 weight_movement_bytes=220200960 total_nvlink_bytes=848070616 weight_to_token_ratio=0.350711 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.287416545e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5847515536 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2924878224 ingress_bytes=4904445952 link_bytes=4904445952 total_nvlink_bytes=7829324176 weight_to_token_ratio=0.338915 prefetch_share_of_ingress=0.202042 dedup_ratio=0.272855 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.529638153e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51210 payload_rows_remote=45908 dup_rows=14326 dup_rows_remote=13368 dedup_ratio=0.218597 router_tokens_per_expert=[0, 19, 0, 636, 516, 34, 0, 13, 0, 79, 33, 4, 0, 514, 337, 14, 0, 1, 10, 28, 69, 17, 0, 2, 1398, 5, 348, 3528, 1, 0, 4, 1374, 3244, 4, 37, 1940, 485, 0, 0, 89, 194, 48, 0, 1794, 4088, 0, 117, 1877, 104, 2, 682, 0, 1285, 0, 1, 0, 1334, 0, 0, 2957, 0, 1086, 506, 1910] payload_rows_per_dst=[7376, 8184, 6540, 5302, 7756, 6646, 3754, 5652] dup_rows_per_dst=[2116, 1954, 1004, 958, 3842, 2706, 394, 1352] combine_rows_per_src=[7376, 8184, 6540, 5302, 7756, 6646, 3754, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=293429248 dispatch_meta_bytes=209504 combine_payload_bytes=293429248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293638752 ingress_bytes=403529728 link_bytes=403529728 token_bytes=587068000 weight_movement_bytes=110100480 total_nvlink_bytes=697168480 weight_to_token_ratio=0.187543 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.018045662e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6130159976 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3066212712 ingress_bytes=4935655424 link_bytes=4935655424 total_nvlink_bytes=8001868136 weight_to_token_ratio=0.305328 prefetch_share_of_ingress=0.200764 dedup_ratio=0.249731 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.583916558e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45258 payload_rows_remote=37620 dup_rows=20278 dup_rows_remote=16278 dedup_ratio=0.309418 router_tokens_per_expert=[81, 22, 3, 2021, 117, 241, 0, 14, 1, 477, 225, 1, 0, 1879, 46, 20, 0, 2, 24, 3, 165, 10, 1, 26, 267, 186, 25, 3577, 2, 1, 0, 290, 3201, 2, 9, 378, 1410, 63, 0, 61, 1, 29, 4, 231, 4062, 1, 45, 1298, 26, 85, 373, 0, 2279, 1, 441, 4, 2279, 0, 0, 3309, 4, 1366, 369, 1710] payload_rows_per_dst=[7828, 8174, 7638, 4316, 7688, 3104, 5526, 984] dup_rows_per_dst=[4324, 5248, 4000, 534, 5080, 114, 892, 86] combine_rows_per_src=[7828, 8174, 7638, 4316, 7688, 3104, 5526, 984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6163029536 prefetch_bytes=1211105280 reduce_bytes=770703360 weight_movement_bytes=1981808640 egress_bytes=3082667552 ingress_bytes=5062170624 link_bytes=5062170624 total_nvlink_bytes=8144838176 weight_to_token_ratio=0.321564 prefetch_share_of_ingress=0.239246 dedup_ratio=0.251230 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=7 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039338 gemm_inner_pad_upper_bound=12827 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.803947299e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=329068544 dispatch_meta_bytes=237104 combine_payload_bytes=329068544 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=329305648 ingress_bytes=549269504 link_bytes=549269504 token_bytes=658374192 weight_movement_bytes=220200960 total_nvlink_bytes=878575152 weight_to_token_ratio=0.334462 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.552700068e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5888081136 prefetch_bytes=1101004800 reduce_bytes=1761607680 weight_movement_bytes=2862612480 egress_bytes=2945158384 ingress_bytes=5805535232 link_bytes=5805535232 total_nvlink_bytes=8750693616 weight_to_token_ratio=0.486171 prefetch_share_of_ingress=0.189647 dedup_ratio=0.272885 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=16 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.009678061e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=269660160 dispatch_meta_bytes=215592 combine_payload_bytes=269660160 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=269875752 ingress_bytes=489861120 link_bytes=489861120 token_bytes=539535912 weight_movement_bytes=220200960 total_nvlink_bytes=759736872 weight_to_token_ratio=0.408130 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=24064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.519490560e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6134907816 prefetch_bytes=770703360 reduce_bytes=1651507200 weight_movement_bytes=2422210560 egress_bytes=3068580776 ingress_bytes=5488537600 link_bytes=5488537600 total_nvlink_bytes=8557118376 weight_to_token_ratio=0.394824 prefetch_share_of_ingress=0.140421 dedup_ratio=0.248788 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=15 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.545469596e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5878624840 prefetch_bytes=1211105280 reduce_bytes=550502400 weight_movement_bytes=1761607680 egress_bytes=2940447304 ingress_bytes=4699785216 link_bytes=4699785216 total_nvlink_bytes=7640232520 weight_to_token_ratio=0.299663 prefetch_share_of_ingress=0.257694 dedup_ratio=0.279321 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=5 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.173699472e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none + [2026-08-26 18:09:51] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 368.3 | throughput per GPU (TFLOP/s/GPU): 335.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.533127E+00 | loss scale: 1.0 | grad norm: 31.978 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44726 payload_rows_remote=38400 dup_rows=20810 dup_rows_remote=17084 dedup_ratio=0.317535 router_tokens_per_expert=[538, 106, 579, 23, 15, 21, 1200, 252, 722, 410, 122, 278, 266, 200, 5, 263, 277, 38, 1329, 102, 363, 379, 118, 20, 600, 1110, 161, 1316, 190, 837, 314, 11, 490, 141, 777, 1422, 65, 0, 58, 399, 791, 725, 1332, 53, 1604, 426, 385, 590, 554, 1029, 292, 62, 529, 1214, 1320, 2348, 1007, 664, 165, 41, 526, 1040, 458, 96] payload_rows_per_dst=[4766, 4402, 5014, 4832, 6224, 6478, 6684, 6326] dup_rows_per_dst=[1186, 1510, 1760, 1614, 3112, 3812, 4090, 3726] combine_rows_per_src=[4766, 4402, 5014, 4832, 6224, 6478, 6684, 6326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45102 payload_rows_remote=39148 dup_rows=20434 dup_rows_remote=17094 dedup_ratio=0.311798 router_tokens_per_expert=[319, 112, 455, 16, 105, 54, 1113, 524, 1022, 489, 179, 171, 188, 222, 19, 452, 146, 208, 1370, 108, 332, 63, 265, 83, 805, 554, 220, 688, 269, 1109, 585, 55, 368, 398, 263, 1459, 109, 22, 117, 607, 709, 901, 881, 115, 1844, 920, 691, 430, 594, 1380, 330, 88, 553, 689, 1758, 1836, 716, 546, 274, 49, 330, 1062, 290, 139] payload_rows_per_dst=[6012, 6108, 6090, 5288, 5506, 5954, 4632, 5512] dup_rows_per_dst=[3056, 2892, 2748, 1906, 2556, 3340, 1192, 2744] combine_rows_per_src=[6012, 6108, 6090, 5288, 5506, 5954, 4632, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45336 payload_rows_remote=39368 dup_rows=20200 dup_rows_remote=17274 dedup_ratio=0.308228 router_tokens_per_expert=[382, 64, 388, 42, 118, 18, 1157, 612, 556, 405, 242, 382, 243, 291, 29, 396, 117, 200, 1348, 96, 374, 83, 120, 27, 1151, 984, 195, 1374, 64, 1258, 392, 70, 471, 83, 677, 1279, 286, 39, 44, 194, 751, 1364, 1081, 231, 1281, 758, 133, 363, 411, 1165, 723, 37, 800, 1300, 1570, 1412, 794, 552, 323, 44, 314, 766, 235, 109] payload_rows_per_dst=[5684, 5682, 5372, 5652, 5968, 6294, 6058, 4626] dup_rows_per_dst=[2702, 2546, 1920, 2576, 2926, 3068, 2814, 1648] combine_rows_per_src=[5684, 5682, 5372, 5652, 5968, 6294, 6058, 4626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44842 payload_rows_remote=39184 dup_rows=20694 dup_rows_remote=18242 dedup_ratio=0.315765 router_tokens_per_expert=[354, 201, 1105, 16, 48, 12, 1318, 691, 932, 562, 226, 172, 172, 173, 14, 402, 104, 205, 1567, 288, 340, 240, 110, 90, 705, 1080, 277, 1133, 285, 825, 384, 40, 471, 54, 591, 1052, 117, 3, 41, 462, 358, 1381, 952, 162, 1328, 681, 261, 429, 448, 897, 502, 47, 519, 661, 1402, 1085, 1166, 1068, 212, 76, 820, 813, 542, 96] payload_rows_per_dst=[5982, 5658, 5788, 6054, 5340, 4902, 4886, 6232] dup_rows_per_dst=[3678, 2452, 2756, 3290, 2260, 1642, 1262, 3354] combine_rows_per_src=[5982, 5658, 5788, 6054, 5340, 4902, 4886, 6232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45804 payload_rows_remote=40860 dup_rows=19732 dup_rows_remote=17722 dedup_ratio=0.301086 router_tokens_per_expert=[322, 100, 442, 23, 23, 39, 1441, 405, 768, 746, 194, 151, 401, 203, 6, 427, 207, 162, 1083, 135, 241, 122, 46, 64, 688, 819, 175, 1025, 248, 1020, 369, 17, 537, 62, 593, 1762, 99, 4, 65, 212, 885, 732, 1221, 70, 1632, 604, 381, 459, 1255, 924, 506, 32, 627, 1057, 2288, 1252, 865, 812, 268, 53, 364, 626, 322, 87] payload_rows_per_dst=[5572, 6784, 5426, 5030, 6202, 5934, 4944, 5912] dup_rows_per_dst=[2522, 3584, 1958, 1642, 2516, 2770, 2010, 2730] combine_rows_per_src=[5572, 6784, 5426, 5030, 6202, 5934, 4944, 5912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45292 payload_rows_remote=39590 dup_rows=20244 dup_rows_remote=17606 dedup_ratio=0.308899 router_tokens_per_expert=[231, 86, 695, 77, 166, 31, 2022, 366, 324, 505, 217, 267, 167, 119, 14, 331, 190, 306, 1323, 116, 252, 102, 165, 25, 840, 1288, 255, 1062, 110, 1186, 472, 19, 365, 67, 744, 1510, 141, 11, 60, 585, 655, 593, 954, 48, 1756, 432, 359, 287, 737, 1231, 833, 21, 770, 1244, 1724, 1077, 1104, 589, 179, 25, 273, 566, 424, 75] payload_rows_per_dst=[6252, 5502, 5826, 5702, 6116, 4902, 6194, 4798] dup_rows_per_dst=[3250, 1834, 2644, 2638, 2974, 1754, 3478, 1672] combine_rows_per_src=[6252, 5502, 5826, 5702, 6116, 4902, 6194, 4798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45344 payload_rows_remote=39686 dup_rows=20192 dup_rows_remote=17712 dedup_ratio=0.308105 router_tokens_per_expert=[342, 144, 466, 13, 19, 38, 1119, 464, 760, 273, 191, 486, 246, 213, 5, 474, 148, 182, 1264, 125, 195, 201, 102, 153, 688, 788, 321, 1022, 83, 768, 420, 25, 445, 139, 408, 1043, 153, 21, 126, 776, 525, 1376, 1108, 74, 1699, 858, 283, 537, 500, 1416, 421, 55, 846, 1207, 1397, 1219, 1029, 773, 97, 89, 271, 1528, 404, 207] payload_rows_per_dst=[5296, 5632, 5658, 5216, 5602, 5956, 6064, 5920] dup_rows_per_dst=[2352, 2458, 2480, 1864, 2664, 2672, 2826, 2876] combine_rows_per_src=[5296, 5632, 5658, 5216, 5602, 5956, 6064, 5920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275251200 dispatch_meta_bytes=221936 combine_payload_bytes=275251200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=275473136 ingress_bytes=385351680 link_bytes=385351680 token_bytes=550724336 weight_movement_bytes=110100480 total_nvlink_bytes=660824816 weight_to_token_ratio=0.199919 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.701899510e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280612864 dispatch_meta_bytes=224968 combine_payload_bytes=280612864 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280837832 ingress_bytes=610914304 link_bytes=610914304 token_bytes=561450696 weight_movement_bytes=330301440 total_nvlink_bytes=891752136 weight_to_token_ratio=0.588300 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062480453e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43758 payload_rows_remote=38646 dup_rows=21778 dup_rows_remote=19664 dedup_ratio=0.332306 router_tokens_per_expert=[310, 145, 757, 24, 22, 33, 1162, 377, 312, 796, 366, 405, 176, 230, 9, 470, 113, 246, 1719, 117, 789, 92, 160, 101, 975, 524, 290, 1634, 211, 701, 603, 30, 119, 54, 376, 1044, 301, 37, 137, 717, 632, 922, 1148, 115, 1711, 426, 264, 662, 1239, 884, 147, 65, 614, 1260, 1482, 783, 990, 896, 395, 65, 600, 310, 296, 178] payload_rows_per_dst=[5112, 5640, 6310, 6912, 4168, 4536, 5864, 5216] dup_rows_per_dst=[2114, 2852, 3786, 5320, 1402, 1522, 2538, 2244] combine_rows_per_src=[5112, 5640, 6310, 6912, 4168, 4536, 5864, 5216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282189824 dispatch_meta_bytes=226568 combine_payload_bytes=282189824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282416392 ingress_bytes=392290304 link_bytes=392290304 token_bytes=564606216 weight_movement_bytes=110100480 total_nvlink_bytes=674706696 weight_to_token_ratio=0.195004 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.822573593e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280870912 dispatch_meta_bytes=229704 combine_payload_bytes=280870912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281100616 ingress_bytes=390971392 link_bytes=390971392 token_bytes=561971528 weight_movement_bytes=110100480 total_nvlink_bytes=672072008 weight_to_token_ratio=0.195918 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.799635544e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=292884480 dispatch_meta_bytes=234328 combine_payload_bytes=292884480 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=293118808 ingress_bytes=733286400 link_bytes=733286400 token_bytes=586003288 weight_movement_bytes=440401920 total_nvlink_bytes=1026405208 weight_to_token_ratio=0.751535 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=8704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.275305654e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=283781120 dispatch_meta_bytes=228784 combine_payload_bytes=283781120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=284009904 ingress_bytes=503982080 link_bytes=503982080 token_bytes=567791024 weight_movement_bytes=220200960 total_nvlink_bytes=787991984 weight_to_token_ratio=0.387820 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.765077279e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284469248 dispatch_meta_bytes=229592 combine_payload_bytes=284469248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=284698840 ingress_bytes=394569728 link_bytes=394569728 token_bytes=569168088 weight_movement_bytes=110100480 total_nvlink_bytes=679268568 weight_to_token_ratio=0.193441 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.862216526e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277014528 dispatch_meta_bytes=233240 combine_payload_bytes=277014528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277247768 ingress_bytes=387115008 link_bytes=387115008 token_bytes=554262296 weight_movement_bytes=110100480 total_nvlink_bytes=664362776 weight_to_token_ratio=0.198643 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.732566684e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47286 payload_rows_remote=39766 dup_rows=18250 dup_rows_remote=13450 dedup_ratio=0.278473 router_tokens_per_expert=[0, 313, 4063, 1427, 5, 16, 2255, 127, 269, 1245, 850, 0, 322, 6, 2, 13, 1, 145, 355, 197, 9, 1785, 0, 39, 501, 962, 1, 28, 2670, 0, 185, 0, 1982, 317, 0, 0, 0, 4, 0, 0, 7, 0, 3861, 5, 243, 1, 1, 2, 524, 965, 8, 283, 3, 609, 0, 2, 106, 0, 2444, 0, 51, 1516, 1349, 694] payload_rows_per_dst=[3430, 4898, 4360, 7040, 8160, 7806, 4072, 7520] dup_rows_per_dst=[346, 1094, 702, 3658, 4572, 2362, 716, 4800] combine_rows_per_src=[3430, 4898, 4360, 7040, 8160, 7806, 4072, 7520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49024 payload_rows_remote=41274 dup_rows=16512 dup_rows_remote=16014 dedup_ratio=0.251953 router_tokens_per_expert=[0, 652, 3857, 783, 1, 38, 2797, 114, 264, 1311, 1129, 2, 425, 4, 0, 129, 0, 53, 196, 355, 6, 1177, 0, 29, 235, 573, 19, 13, 2561, 13, 118, 1, 1626, 507, 0, 0, 3, 0, 0, 4, 12, 0, 3860, 1, 247, 4, 0, 0, 262, 938, 6, 209, 29, 1773, 0, 1, 46, 0, 2593, 0, 14, 868, 2390, 520] payload_rows_per_dst=[4746, 6204, 3288, 5934, 7944, 7750, 6650, 6508] dup_rows_per_dst=[1242, 3106, 344, 1132, 4050, 498, 3354, 2786] combine_rows_per_src=[4746, 6204, 3288, 5934, 7944, 7750, 6650, 6508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49476 payload_rows_remote=44736 dup_rows=16060 dup_rows_remote=15236 dedup_ratio=0.245056 router_tokens_per_expert=[0, 272, 4037, 970, 1, 176, 2310, 60, 482, 1324, 626, 5, 305, 4, 0, 36, 0, 51, 1051, 96, 3, 1053, 5, 26, 405, 975, 6, 25, 2768, 2, 200, 2, 2385, 576, 0, 0, 11, 5, 0, 0, 26, 0, 3539, 4, 110, 0, 4, 0, 601, 462, 0, 176, 3, 1057, 0, 1, 7, 0, 2133, 1, 67, 2170, 1064, 1090] payload_rows_per_dst=[7154, 4740, 5982, 6536, 5294, 7124, 6314, 6332] dup_rows_per_dst=[4700, 824, 2386, 2230, 660, 242, 2552, 2466] combine_rows_per_src=[7154, 4740, 5982, 6536, 5294, 7124, 6314, 6332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50532 payload_rows_remote=43972 dup_rows=15004 dup_rows_remote=13416 dedup_ratio=0.228943 router_tokens_per_expert=[0, 1349, 4038, 1158, 3, 14, 1454, 95, 922, 1364, 1295, 0, 627, 2, 0, 15, 0, 48, 608, 56, 19, 1546, 0, 39, 358, 375, 3, 13, 2786, 21, 122, 9, 2986, 338, 0, 0, 0, 5, 0, 0, 8, 0, 3726, 0, 119, 1, 0, 0, 1233, 649, 1, 204, 0, 1195, 2, 10, 12, 0, 1058, 0, 16, 1644, 757, 465] payload_rows_per_dst=[5890, 6254, 7558, 6202, 6560, 7492, 5924, 4652] dup_rows_per_dst=[2256, 2196, 3660, 1172, 1588, 216, 2780, 1136] combine_rows_per_src=[5890, 6254, 7558, 6202, 6560, 7492, 5924, 4652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48100 payload_rows_remote=44092 dup_rows=17436 dup_rows_remote=16726 dedup_ratio=0.266052 router_tokens_per_expert=[0, 336, 4007, 1335, 0, 3, 2559, 101, 272, 1992, 1124, 0, 419, 4, 0, 19, 0, 67, 308, 85, 0, 1807, 0, 38, 410, 505, 0, 8, 2733, 12, 202, 0, 2779, 242, 2, 0, 17, 2, 0, 0, 49, 1, 3785, 2, 95, 0, 0, 0, 393, 597, 5, 449, 0, 900, 0, 15, 4, 0, 2035, 0, 22, 1087, 1238, 703] payload_rows_per_dst=[3248, 7606, 4228, 6252, 8142, 7650, 4008, 6966] dup_rows_per_dst=[302, 5172, 382, 1488, 5956, 214, 710, 3212] combine_rows_per_src=[3248, 7606, 4228, 6252, 8142, 7650, 4008, 6966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48888 payload_rows_remote=43118 dup_rows=16648 dup_rows_remote=14880 dedup_ratio=0.254028 router_tokens_per_expert=[0, 257, 3939, 1710, 7, 19, 1471, 505, 856, 1535, 1001, 5, 641, 1, 2, 287, 0, 95, 1125, 46, 24, 1526, 0, 82, 523, 735, 1, 0, 2252, 0, 257, 1, 1582, 450, 0, 0, 24, 3, 3, 3, 84, 0, 3852, 1, 125, 1, 0, 0, 659, 688, 12, 281, 8, 968, 13, 58, 94, 0, 1535, 4, 40, 1763, 1227, 387] payload_rows_per_dst=[5708, 6200, 8044, 5770, 3916, 7722, 6150, 5378] dup_rows_per_dst=[2230, 2456, 5630, 1768, 214, 404, 2294, 1652] combine_rows_per_src=[5708, 6200, 8044, 5770, 3916, 7722, 6150, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49646 payload_rows_remote=41594 dup_rows=15890 dup_rows_remote=10884 dedup_ratio=0.242462 router_tokens_per_expert=[0, 427, 3913, 1128, 9, 10, 2399, 45, 704, 1601, 1009, 1, 254, 1, 21, 31, 4, 216, 209, 65, 0, 2022, 2, 98, 1092, 288, 0, 35, 2353, 0, 246, 3, 1798, 266, 0, 6, 0, 3, 0, 0, 106, 0, 3585, 1, 124, 0, 0, 0, 454, 771, 0, 578, 3, 696, 1, 32, 82, 0, 2577, 6, 32, 1971, 1348, 142] payload_rows_per_dst=[6106, 5854, 8052, 6000, 3972, 7256, 6782, 5624] dup_rows_per_dst=[1930, 1390, 5006, 2034, 174, 376, 3442, 1538] combine_rows_per_src=[6106, 5854, 8052, 6000, 3972, 7256, 6782, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=285042688 dispatch_meta_bytes=212864 combine_payload_bytes=285042688 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=285255552 ingress_bytes=505243648 link_bytes=505243648 token_bytes=570298240 weight_movement_bytes=220200960 total_nvlink_bytes=790499200 weight_to_token_ratio=0.386115 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.787018022e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=295852032 dispatch_meta_bytes=229152 combine_payload_bytes=295852032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=296081184 ingress_bytes=405952512 link_bytes=405952512 token_bytes=591933216 weight_movement_bytes=110100480 total_nvlink_bytes=702033696 weight_to_token_ratio=0.186002 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.060181861e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=320667648 dispatch_meta_bytes=239888 combine_payload_bytes=320667648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=320907536 ingress_bytes=430768128 link_bytes=430768128 token_bytes=641575184 weight_movement_bytes=110100480 total_nvlink_bytes=751675664 weight_to_token_ratio=0.171610 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.491766238e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=315191296 dispatch_meta_bytes=229552 combine_payload_bytes=315191296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=315420848 ingress_bytes=425291776 link_bytes=425291776 token_bytes=630612144 weight_movement_bytes=110100480 total_nvlink_bytes=740712624 weight_to_token_ratio=0.174593 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.396523469e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=316051456 dispatch_meta_bytes=243272 combine_payload_bytes=316051456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316294728 ingress_bytes=426151936 link_bytes=426151936 token_bytes=632346184 weight_movement_bytes=110100480 total_nvlink_bytes=742446664 weight_to_token_ratio=0.174114 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.411483067e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=309069824 dispatch_meta_bytes=231992 combine_payload_bytes=309069824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=309301816 ingress_bytes=419170304 link_bytes=419170304 token_bytes=618371640 weight_movement_bytes=110100480 total_nvlink_bytes=728472120 weight_to_token_ratio=0.178049 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.290061003e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47416 payload_rows_remote=39258 dup_rows=18120 dup_rows_remote=10030 dedup_ratio=0.276489 router_tokens_per_expert=[0, 667, 3992, 1268, 3, 20, 1577, 233, 477, 1605, 966, 0, 470, 9, 1, 243, 1, 204, 926, 23, 0, 1794, 0, 9, 523, 1133, 0, 9, 2363, 4, 126, 2, 1853, 311, 0, 0, 2, 1, 0, 0, 7, 0, 3781, 2, 421, 0, 1, 0, 595, 1239, 4, 183, 7, 964, 0, 0, 44, 0, 1902, 1, 106, 1267, 1200, 229] payload_rows_per_dst=[8158, 5860, 4812, 6152, 4154, 7656, 6450, 4174] dup_rows_per_dst=[8090, 1682, 1102, 2168, 180, 768, 3338, 792] combine_rows_per_src=[8158, 5860, 4812, 6152, 4154, 7656, 6450, 4174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=298145792 dispatch_meta_bytes=209912 combine_payload_bytes=298145792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298355704 ingress_bytes=408246272 link_bytes=408246272 token_bytes=596501496 weight_movement_bytes=110100480 total_nvlink_bytes=706601976 weight_to_token_ratio=0.184577 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.100074120e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54570 payload_rows_remote=47722 dup_rows=10966 dup_rows_remote=9714 dedup_ratio=0.167328 router_tokens_per_expert=[343, 3876, 709, 19, 34, 0, 14, 0, 13, 14, 5, 3, 62, 57, 3981, 0, 1, 13, 4, 3947, 3, 1, 1, 0, 0, 751, 1, 720, 0, 10, 1, 743, 92, 3282, 3, 1583, 2, 4, 816, 1831, 179, 5, 0, 3, 4003, 93, 1, 1, 3, 1, 4075, 3, 678, 0, 16, 0, 3, 10, 18, 2, 729, 1, 0, 5] payload_rows_per_dst=[2188, 6972, 7912, 7952, 6468, 8054, 8176, 6848] dup_rows_per_dst=[50, 204, 28, 4252, 2194, 516, 2470, 1252] combine_rows_per_src=[2188, 6972, 7912, 7952, 6468, 8054, 8176, 6848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=281401344 dispatch_meta_bytes=197152 combine_payload_bytes=281401344 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=281598496 ingress_bytes=942004224 link_bytes=942004224 token_bytes=562999840 weight_movement_bytes=660602880 total_nvlink_bytes=1223602720 weight_to_token_ratio=1.173362 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=10624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.638300279e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54444 payload_rows_remote=46598 dup_rows=11092 dup_rows_remote=10078 dedup_ratio=0.169250 router_tokens_per_expert=[731, 3739, 219, 65, 154, 0, 27, 1, 2, 1, 1, 0, 111, 43, 4050, 1, 0, 4, 0, 3947, 0, 0, 2, 2, 21, 472, 2, 882, 0, 36, 0, 364, 291, 3014, 0, 1115, 0, 0, 422, 1287, 217, 149, 2, 0, 3835, 219, 8, 0, 0, 0, 4053, 0, 992, 0, 1, 0, 0, 0, 32, 0, 2254, 0, 0, 0] payload_rows_per_dst=[2234, 8104, 7900, 7914, 5154, 7846, 8120, 7172] dup_rows_per_dst=[160, 314, 10, 3118, 1076, 1014, 1972, 3428] combine_rows_per_src=[2234, 8104, 7900, 7914, 5154, 7846, 8120, 7172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54746 payload_rows_remote=48218 dup_rows=10790 dup_rows_remote=8436 dedup_ratio=0.164642 router_tokens_per_expert=[216, 3732, 1130, 105, 230, 0, 126, 0, 0, 2, 0, 0, 24, 73, 3667, 0, 1, 1, 0, 3980, 0, 1, 0, 0, 1, 701, 0, 1098, 0, 24, 0, 224, 144, 3041, 0, 2172, 0, 0, 1048, 1077, 210, 0, 0, 0, 3416, 35, 1, 0, 0, 0, 3990, 0, 905, 0, 29, 0, 0, 0, 481, 1, 882, 0, 0, 0] payload_rows_per_dst=[3310, 7350, 7964, 7826, 6528, 6942, 8000, 6826] dup_rows_per_dst=[304, 182, 2, 3734, 2354, 382, 1848, 1984] combine_rows_per_src=[3310, 7350, 7964, 7826, 6528, 6942, 8000, 6826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50676 payload_rows_remote=42710 dup_rows=14860 dup_rows_remote=14548 dedup_ratio=0.226746 router_tokens_per_expert=[273, 3827, 801, 74, 230, 0, 93, 1, 12, 16, 0, 4, 87, 59, 3961, 0, 15, 12, 0, 3691, 6, 1, 0, 0, 7, 515, 0, 557, 1, 32, 3, 486, 64, 3329, 0, 1639, 30, 3, 1092, 1303, 83, 0, 0, 1, 3577, 58, 44, 1, 1, 0, 4066, 4, 1474, 0, 12, 1, 0, 0, 29, 4, 1158, 0, 0, 31] payload_rows_per_dst=[8146, 7966, 7482, 2962, 6192, 7278, 3860, 6790] dup_rows_per_dst=[9248, 312, 44, 240, 2530, 250, 460, 1776] combine_rows_per_src=[8146, 7966, 7482, 2962, 6192, 7278, 3860, 6790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54146 payload_rows_remote=46012 dup_rows=11390 dup_rows_remote=8886 dedup_ratio=0.173798 router_tokens_per_expert=[564, 3822, 501, 22, 153, 0, 34, 0, 1, 14, 0, 1, 121, 57, 3978, 0, 0, 2, 1, 3809, 3, 11, 0, 0, 3, 664, 1, 870, 0, 8, 1, 502, 339, 3250, 0, 1096, 0, 0, 968, 778, 201, 2, 0, 0, 3961, 131, 0, 0, 0, 0, 4064, 0, 1186, 0, 69, 0, 0, 2, 152, 3, 1419, 0, 4, 0] payload_rows_per_dst=[2326, 7996, 7640, 7924, 5032, 7946, 8134, 7148] dup_rows_per_dst=[222, 348, 12, 3818, 1330, 644, 2504, 2512] combine_rows_per_src=[2326, 7996, 7640, 7924, 5032, 7946, 8134, 7148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=342071296 dispatch_meta_bytes=229744 combine_payload_bytes=342071296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=342301040 ingress_bytes=452171776 link_bytes=452171776 token_bytes=684372336 weight_movement_bytes=110100480 total_nvlink_bytes=794472816 weight_to_token_ratio=0.160878 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.864010879e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55392 payload_rows_remote=49770 dup_rows=10144 dup_rows_remote=8030 dedup_ratio=0.154785 router_tokens_per_expert=[630, 3832, 763, 8, 226, 0, 66, 0, 2, 5, 1, 0, 183, 243, 3745, 0, 0, 0, 0, 3804, 0, 4, 0, 1, 22, 455, 1, 1523, 0, 132, 0, 426, 447, 2515, 0, 2003, 2, 25, 415, 521, 97, 1, 1, 0, 3803, 105, 3, 0, 0, 0, 4029, 1, 635, 1, 9, 1, 0, 0, 413, 6, 1663, 0, 0, 0] payload_rows_per_dst=[6296, 7728, 7616, 5622, 5686, 7698, 8076, 6670] dup_rows_per_dst=[2136, 630, 2, 2114, 1140, 322, 1276, 2524] combine_rows_per_src=[6296, 7728, 7616, 5622, 5686, 7698, 8076, 6670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56418 payload_rows_remote=48800 dup_rows=9118 dup_rows_remote=9116 dedup_ratio=0.139130 router_tokens_per_expert=[492, 3783, 721, 35, 249, 0, 19, 0, 0, 35, 1, 9, 95, 43, 3960, 3, 0, 1, 0, 3807, 1, 1, 0, 0, 0, 523, 1, 1045, 0, 51, 0, 324, 158, 3058, 0, 915, 0, 15, 1340, 1941, 198, 4, 0, 2, 3922, 36, 0, 0, 0, 0, 4073, 0, 540, 0, 19, 0, 0, 0, 64, 2, 1254, 4, 0, 24] payload_rows_per_dst=[7924, 7958, 7618, 3468, 6624, 7872, 8148, 6806] dup_rows_per_dst=[2674, 334, 2, 420, 2114, 452, 1116, 2006] combine_rows_per_src=[7924, 7958, 7618, 3468, 6624, 7872, 8148, 6806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=334014464 dispatch_meta_bytes=226704 combine_payload_bytes=334014464 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=334241168 ingress_bytes=444114944 link_bytes=444114944 token_bytes=668255632 weight_movement_bytes=110100480 total_nvlink_bytes=778356112 weight_to_token_ratio=0.164758 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=30464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.723889320e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=345626624 dispatch_meta_bytes=226616 combine_payload_bytes=345626624 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=345853240 ingress_bytes=786028544 link_bytes=786028544 token_bytes=691479864 weight_movement_bytes=440401920 total_nvlink_bytes=1131881784 weight_to_token_ratio=0.636898 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.367032917e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=306145280 dispatch_meta_bytes=229032 combine_payload_bytes=306145280 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=306374312 ingress_bytes=526346240 link_bytes=526346240 token_bytes=612519592 weight_movement_bytes=220200960 total_nvlink_bytes=832720552 weight_to_token_ratio=0.359500 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.154026804e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=329814016 dispatch_meta_bytes=219592 combine_payload_bytes=329814016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=330033608 ingress_bytes=550014976 link_bytes=550014976 token_bytes=659847624 weight_movement_bytes=220200960 total_nvlink_bytes=880048584 weight_to_token_ratio=0.333715 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=24960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.565665052e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=356751360 dispatch_meta_bytes=231200 combine_payload_bytes=356751360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=356982560 ingress_bytes=466851840 link_bytes=466851840 token_bytes=713733920 weight_movement_bytes=110100480 total_nvlink_bytes=823834400 weight_to_token_ratio=0.154260 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.119321337e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=349798400 dispatch_meta_bytes=231664 combine_payload_bytes=349798400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=350030064 ingress_bytes=459898880 link_bytes=459898880 token_bytes=699828464 weight_movement_bytes=110100480 total_nvlink_bytes=809928944 weight_to_token_ratio=0.157325 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.998397927e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46370 payload_rows_remote=38178 dup_rows=19166 dup_rows_remote=9040 dedup_ratio=0.292450 router_tokens_per_expert=[373, 3881, 588, 19, 176, 0, 56, 0, 0, 14, 0, 0, 55, 197, 4040, 0, 1, 42, 0, 3832, 0, 4, 0, 0, 0, 904, 0, 1025, 0, 84, 1, 72, 237, 3056, 0, 2036, 0, 0, 1048, 1155, 134, 53, 0, 0, 3951, 82, 1, 0, 0, 3, 4066, 0, 659, 0, 26, 0, 0, 0, 199, 2, 696, 0, 0, 0] payload_rows_per_dst=[8192, 8108, 7934, 3708, 7450, 7852, 1372, 1754] dup_rows_per_dst=[10126, 1030, 3370, 464, 3664, 468, 4, 40] combine_rows_per_src=[8192, 8108, 7934, 3708, 7450, 7852, 1372, 1754] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48036 payload_rows_remote=39878 dup_rows=17500 dup_rows_remote=6970 dedup_ratio=0.267029 router_tokens_per_expert=[1, 0, 20, 2529, 1504, 2, 9, 0, 1, 0, 1, 0, 9, 4041, 0, 6, 13, 107, 0, 1, 0, 0, 9, 61, 7, 2, 0, 633, 2, 0, 0, 123, 0, 2, 6, 3, 0, 4095, 2023, 0, 5, 7, 2, 1, 1801, 2, 13, 0, 3774, 7, 2, 12, 2, 0, 1294, 10, 4065, 4090, 844, 1, 2, 5, 1618, 1] payload_rows_per_dst=[6642, 8108, 5970, 1524, 4054, 8190, 5390, 8158] dup_rows_per_dst=[1488, 8, 356, 10, 14, 3662, 1432, 10530] combine_rows_per_src=[6642, 8108, 5970, 1524, 4054, 8190, 5390, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273659904 dispatch_meta_bytes=188872 combine_payload_bytes=273659904 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=273848776 ingress_bytes=493860864 link_bytes=493860864 token_bytes=547508680 weight_movement_bytes=220200960 total_nvlink_bytes=767709640 weight_to_token_ratio=0.402187 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=14080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.589052687e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55140 payload_rows_remote=46948 dup_rows=10396 dup_rows_remote=6532 dedup_ratio=0.158630 router_tokens_per_expert=[2, 9, 3, 2319, 1287, 0, 13, 0, 1, 0, 0, 0, 1, 4093, 1, 0, 1, 108, 0, 1, 0, 0, 39, 21, 0, 10, 0, 782, 0, 0, 15, 305, 0, 0, 0, 9, 0, 4093, 1686, 5, 2, 1, 0, 0, 1919, 0, 13, 0, 3028, 0, 0, 1, 0, 0, 1062, 0, 4087, 4070, 1471, 0, 0, 3, 2307, 0] payload_rows_per_dst=[6156, 8188, 8148, 8182, 3398, 8192, 6852, 6024] dup_rows_per_dst=[1110, 4, 332, 2216, 2, 3864, 1330, 1538] combine_rows_per_src=[6156, 8188, 8148, 8182, 3398, 8192, 6852, 6024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55342 payload_rows_remote=52032 dup_rows=10194 dup_rows_remote=10190 dedup_ratio=0.155548 router_tokens_per_expert=[0, 5, 35, 1127, 2392, 0, 0, 0, 0, 0, 0, 0, 1, 4071, 0, 0, 2, 65, 0, 12, 0, 0, 10, 36, 5, 0, 10, 324, 0, 0, 2, 19, 0, 0, 0, 0, 0, 4096, 1654, 3, 7, 6, 1, 0, 1762, 0, 0, 0, 3732, 0, 3, 0, 0, 0, 2257, 2, 4095, 4070, 1467, 0, 0, 2, 1495, 0] payload_rows_per_dst=[6372, 8142, 8176, 8192, 3310, 8192, 7888, 5070] dup_rows_per_dst=[746, 2, 214, 718, 4, 3552, 4100, 858] combine_rows_per_src=[6372, 8142, 8176, 8192, 3310, 8192, 7888, 5070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51022 payload_rows_remote=42848 dup_rows=14514 dup_rows_remote=14502 dedup_ratio=0.221466 router_tokens_per_expert=[0, 1, 1, 2009, 1764, 0, 0, 0, 2, 0, 12, 1, 3, 4075, 0, 0, 6, 65, 15, 30, 0, 5, 94, 213, 45, 2, 0, 844, 1, 0, 7, 186, 3, 0, 0, 0, 1, 4090, 1221, 1, 2, 0, 0, 0, 1558, 0, 0, 1, 3493, 1, 0, 6, 0, 1, 1252, 0, 4053, 4024, 1442, 1, 0, 0, 2229, 8] payload_rows_per_dst=[6228, 8174, 8188, 8186, 8188, 3122, 3156, 5780] dup_rows_per_dst=[1322, 12, 716, 2090, 8440, 0, 354, 1580] combine_rows_per_src=[6228, 8174, 8188, 8186, 8188, 3122, 3156, 5780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53480 payload_rows_remote=45916 dup_rows=12056 dup_rows_remote=8698 dedup_ratio=0.183960 router_tokens_per_expert=[0, 0, 5, 2180, 1669, 0, 3, 0, 0, 0, 0, 0, 2, 4096, 0, 1, 0, 58, 2, 0, 0, 1, 16, 61, 12, 13, 5, 476, 0, 0, 1, 44, 0, 0, 0, 0, 0, 4094, 2031, 11, 3, 1, 5, 0, 1703, 0, 0, 0, 3568, 0, 37, 0, 0, 0, 1855, 1, 4061, 4083, 1372, 0, 0, 2, 1296, 0] payload_rows_per_dst=[6406, 8192, 8170, 4990, 4076, 8190, 7564, 5892] dup_rows_per_dst=[1308, 6, 272, 744, 8, 3422, 3358, 2938] combine_rows_per_src=[6406, 8192, 8170, 4990, 4076, 8190, 7564, 5892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=285845504 dispatch_meta_bytes=187392 combine_payload_bytes=285845504 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=286032896 ingress_bytes=616146944 link_bytes=616146944 token_bytes=571878400 weight_movement_bytes=330301440 total_nvlink_bytes=902179840 weight_to_token_ratio=0.577573 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.071580874e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55596 payload_rows_remote=47454 dup_rows=9940 dup_rows_remote=8932 dedup_ratio=0.151672 router_tokens_per_expert=[0, 3, 23, 1670, 1717, 40, 90, 0, 0, 1, 1, 8, 0, 4086, 0, 0, 6, 26, 2, 12, 0, 0, 64, 29, 2, 10, 16, 400, 0, 0, 5, 90, 0, 0, 1, 2, 0, 4096, 1823, 3, 32, 8, 0, 0, 2592, 0, 10, 0, 3315, 0, 0, 3, 2, 0, 2234, 1, 4052, 4089, 1705, 0, 0, 5, 494, 0] payload_rows_per_dst=[6150, 8174, 8182, 8142, 6874, 6234, 7664, 4176] dup_rows_per_dst=[936, 18, 274, 1008, 2686, 1340, 3446, 232] combine_rows_per_src=[6150, 8174, 8182, 8142, 6874, 6234, 7664, 4176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=372965376 dispatch_meta_bytes=248888 combine_payload_bytes=372965376 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=373214264 ingress_bytes=593166336 link_bytes=593166336 token_bytes=746179640 weight_movement_bytes=220200960 total_nvlink_bytes=966380600 weight_to_token_ratio=0.295104 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.031613817e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54598 payload_rows_remote=46486 dup_rows=10938 dup_rows_remote=10696 dedup_ratio=0.166901 router_tokens_per_expert=[0, 2, 1, 1904, 1665, 0, 13, 2, 22, 0, 0, 0, 0, 4083, 0, 0, 0, 29, 0, 1, 5, 0, 52, 41, 1, 0, 75, 596, 0, 0, 0, 281, 0, 0, 19, 6, 3, 4086, 1894, 12, 4, 0, 2, 0, 1245, 0, 35, 0, 3433, 0, 0, 0, 0, 0, 1926, 7, 3956, 4049, 2322, 0, 2, 3, 991, 0] payload_rows_per_dst=[6176, 8168, 8112, 8152, 8184, 2548, 7542, 5716] dup_rows_per_dst=[998, 42, 242, 1666, 3856, 24, 3190, 920] combine_rows_per_src=[6176, 8168, 8112, 8152, 8184, 2548, 7542, 5716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=336523264 dispatch_meta_bytes=213920 combine_payload_bytes=336523264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=336737184 ingress_bytes=446623744 link_bytes=446623744 token_bytes=673260448 weight_movement_bytes=110100480 total_nvlink_bytes=783360928 weight_to_token_ratio=0.163533 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.767521478e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=307134464 dispatch_meta_bytes=229400 combine_payload_bytes=307134464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=307363864 ingress_bytes=417234944 link_bytes=417234944 token_bytes=614498328 weight_movement_bytes=110100480 total_nvlink_bytes=724598808 weight_to_token_ratio=0.179171 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.256401910e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=329125888 dispatch_meta_bytes=218456 combine_payload_bytes=329125888 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=329344344 ingress_bytes=769527808 link_bytes=769527808 token_bytes=658470232 weight_movement_bytes=440401920 total_nvlink_bytes=1098872152 weight_to_token_ratio=0.668826 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.338335424e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=340150272 dispatch_meta_bytes=225544 combine_payload_bytes=340150272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=340375816 ingress_bytes=450250752 link_bytes=450250752 token_bytes=680526088 weight_movement_bytes=110100480 total_nvlink_bytes=790626568 weight_to_token_ratio=0.161787 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.830601113e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53080 payload_rows_remote=45208 dup_rows=12456 dup_rows_remote=6830 dedup_ratio=0.190063 router_tokens_per_expert=[0, 0, 8, 2510, 1860, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 9, 0, 2, 1, 0, 4, 3, 0, 18, 0, 548, 0, 0, 2, 196, 0, 0, 0, 2, 1, 4096, 1467, 3, 1, 0, 39, 0, 2446, 0, 16, 0, 3078, 0, 0, 0, 0, 0, 1135, 0, 4094, 4073, 1712, 0, 0, 2, 1348, 0] payload_rows_per_dst=[7872, 8188, 8148, 8190, 8192, 4918, 2270, 5302] dup_rows_per_dst=[5626, 110, 36, 1526, 4250, 86, 0, 822] combine_rows_per_src=[7872, 8188, 8148, 8190, 8192, 4918, 2270, 5302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=333211648 dispatch_meta_bytes=228728 combine_payload_bytes=333211648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=333440376 ingress_bytes=443312128 link_bytes=443312128 token_bytes=666652024 weight_movement_bytes=110100480 total_nvlink_bytes=776752504 weight_to_token_ratio=0.165154 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.709927029e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50852 payload_rows_remote=43682 dup_rows=14684 dup_rows_remote=12430 dedup_ratio=0.224060 router_tokens_per_expert=[0, 358, 1123, 4, 0, 0, 4073, 0, 0, 0, 4, 25, 17, 1168, 0, 0, 0, 0, 4070, 77, 5, 4075, 0, 3920, 3008, 0, 10, 13, 97, 0, 164, 0, 0, 0, 2864, 0, 0, 0, 3249, 4, 34, 7, 922, 9, 1, 0, 3, 0, 0, 3, 0, 16, 1, 0, 0, 0, 32, 0, 0, 14, 1, 82, 0, 3315] payload_rows_per_dst=[2742, 2420, 8172, 6158, 7842, 8170, 8178, 7170] dup_rows_per_dst=[228, 8, 4330, 426, 5404, 1928, 106, 2254] combine_rows_per_src=[2742, 2420, 8172, 6158, 7842, 8170, 8178, 7170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54184 payload_rows_remote=47162 dup_rows=11352 dup_rows_remote=9268 dedup_ratio=0.173218 router_tokens_per_expert=[0, 1060, 311, 0, 0, 0, 4095, 0, 0, 0, 0, 9, 0, 843, 0, 0, 0, 0, 4096, 16, 0, 4096, 0, 4081, 2154, 0, 0, 0, 201, 9, 158, 0, 0, 0, 2142, 0, 0, 0, 2408, 3, 2, 0, 3303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 108, 0, 3651] payload_rows_per_dst=[2558, 8192, 8162, 4656, 7022, 8074, 8192, 7328] dup_rows_per_dst=[300, 1704, 32, 388, 2084, 6610, 0, 234] combine_rows_per_src=[2558, 8192, 8162, 4656, 7022, 8074, 8192, 7328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=324050944 dispatch_meta_bytes=208152 combine_payload_bytes=324050944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=324259096 ingress_bytes=434151424 link_bytes=434151424 token_bytes=648310040 weight_movement_bytes=110100480 total_nvlink_bytes=758410520 weight_to_token_ratio=0.169827 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.550607320e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55070 payload_rows_remote=46880 dup_rows=10466 dup_rows_remote=6208 dedup_ratio=0.159698 router_tokens_per_expert=[0, 1190, 452, 0, 2, 0, 4093, 0, 0, 4, 0, 46, 0, 1043, 0, 0, 0, 0, 4091, 81, 0, 4093, 0, 4071, 1807, 0, 2, 0, 25, 23, 176, 4, 0, 0, 3269, 6, 0, 0, 2156, 3, 11, 0, 2120, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 52, 1, 3942] payload_rows_per_dst=[2994, 8192, 8154, 3914, 7544, 8190, 8188, 7894] dup_rows_per_dst=[294, 2176, 150, 160, 3324, 4258, 6, 98] combine_rows_per_src=[2994, 8192, 8154, 3914, 7544, 8190, 8188, 7894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53062 payload_rows_remote=44986 dup_rows=12474 dup_rows_remote=9694 dedup_ratio=0.190338 router_tokens_per_expert=[0, 702, 444, 4, 2, 9, 4090, 5, 0, 0, 0, 9, 0, 1388, 0, 0, 0, 0, 4031, 38, 1, 4081, 0, 4043, 2982, 0, 3, 0, 142, 5, 125, 0, 0, 0, 2087, 1, 1, 0, 2589, 9, 3, 0, 1918, 2, 0, 0, 0, 7, 3, 0, 4, 7, 0, 0, 0, 0, 22, 1, 3, 0, 15, 22, 4, 3966] payload_rows_per_dst=[8192, 8076, 5120, 7506, 4178, 3856, 8180, 7954] dup_rows_per_dst=[5320, 2780, 44, 4186, 18, 4, 10, 112] combine_rows_per_src=[8192, 8076, 5120, 7506, 4178, 3856, 8180, 7954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=313112576 dispatch_meta_bytes=224448 combine_payload_bytes=313112576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=313337024 ingress_bytes=423213056 link_bytes=423213056 token_bytes=626449600 weight_movement_bytes=110100480 total_nvlink_bytes=736550080 weight_to_token_ratio=0.175753 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.360371110e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=338057216 dispatch_meta_bytes=225720 combine_payload_bytes=338057216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=338282936 ingress_bytes=558258176 link_bytes=558258176 token_bytes=676340152 weight_movement_bytes=220200960 total_nvlink_bytes=896541112 weight_to_token_ratio=0.325577 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.709027858e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50936 payload_rows_remote=42772 dup_rows=14600 dup_rows_remote=14596 dedup_ratio=0.222778 router_tokens_per_expert=[0, 469, 545, 0, 4, 0, 4088, 0, 0, 0, 0, 43, 5, 1127, 0, 5, 0, 0, 4074, 6, 0, 4080, 0, 4062, 2706, 0, 0, 0, 141, 65, 511, 0, 0, 0, 2921, 0, 0, 0, 2194, 10, 8, 2, 1452, 4, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 20, 0, 1, 0, 0, 194, 0, 4024] payload_rows_per_dst=[1880, 3368, 8140, 5930, 7198, 8188, 8164, 8068] dup_rows_per_dst=[156, 204, 6932, 916, 3052, 2926, 4, 410] combine_rows_per_src=[1880, 3368, 8140, 5930, 7198, 8188, 8164, 8068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56946 payload_rows_remote=50824 dup_rows=8590 dup_rows_remote=7550 dedup_ratio=0.131073 router_tokens_per_expert=[0, 747, 1338, 0, 2, 0, 4070, 1, 0, 0, 0, 3, 0, 855, 0, 0, 0, 0, 4092, 12, 0, 4063, 2, 4093, 2429, 0, 0, 0, 338, 80, 240, 0, 1, 0, 1491, 3, 0, 0, 2539, 3, 33, 0, 2339, 2, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 28, 0, 1, 0, 1, 29, 0, 3930] payload_rows_per_dst=[8172, 8190, 8190, 6122, 5570, 4710, 8126, 7866] dup_rows_per_dst=[4144, 1710, 24, 1040, 1516, 42, 2, 112] combine_rows_per_src=[8172, 8190, 8190, 6122, 5570, 4710, 8126, 7866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=322459648 dispatch_meta_bytes=218720 combine_payload_bytes=322459648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=322678368 ingress_bytes=432560128 link_bytes=432560128 token_bytes=645138016 weight_movement_bytes=110100480 total_nvlink_bytes=755238496 weight_to_token_ratio=0.170662 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=25216 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.522932065e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58048 payload_rows_remote=49996 dup_rows=7488 dup_rows_remote=7474 dedup_ratio=0.114258 router_tokens_per_expert=[0, 385, 399, 2, 0, 0, 4095, 0, 0, 0, 0, 15, 0, 864, 0, 2, 0, 0, 4085, 11, 0, 4090, 0, 4022, 2685, 0, 0, 0, 280, 2, 303, 0, 2, 4, 2815, 0, 0, 0, 2409, 24, 1, 0, 2183, 0, 0, 1, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6, 0, 8, 0, 0, 155, 6, 3906] payload_rows_per_dst=[8190, 8176, 8052, 7584, 5632, 4378, 8182, 7854] dup_rows_per_dst=[1572, 1756, 14, 3774, 58, 2, 4, 308] combine_rows_per_src=[8190, 8176, 8052, 7584, 5632, 4378, 8182, 7854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=336035840 dispatch_meta_bytes=212352 combine_payload_bytes=336035840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=336248192 ingress_bytes=446136320 link_bytes=446136320 token_bytes=672284032 weight_movement_bytes=110100480 total_nvlink_bytes=782384512 weight_to_token_ratio=0.163771 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.759044373e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=306589696 dispatch_meta_bytes=229472 combine_payload_bytes=306589696 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=306819168 ingress_bytes=526790656 link_bytes=526790656 token_bytes=613408864 weight_movement_bytes=220200960 total_nvlink_bytes=833609824 weight_to_token_ratio=0.358979 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.161755930e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49480 payload_rows_remote=41288 dup_rows=16056 dup_rows_remote=5454 dedup_ratio=0.244995 router_tokens_per_expert=[0, 531, 692, 0, 3, 0, 4096, 0, 0, 0, 0, 5, 0, 191, 0, 0, 1, 0, 4096, 9, 0, 4095, 0, 4075, 3379, 0, 0, 0, 106, 0, 118, 0, 0, 0, 2547, 0, 0, 0, 2785, 0, 7, 0, 2093, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 33, 0, 0, 1, 1, 18, 0, 3884] payload_rows_per_dst=[8192, 8192, 20, 7826, 5094, 4188, 8190, 7778] dup_rows_per_dst=[10602, 392, 0, 4950, 0, 12, 4, 96] combine_rows_per_src=[8192, 8192, 20, 7826, 5094, 4188, 8190, 7778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=364306432 dispatch_meta_bytes=233496 combine_payload_bytes=364306432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364539928 ingress_bytes=474406912 link_bytes=474406912 token_bytes=728846360 weight_movement_bytes=110100480 total_nvlink_bytes=838946840 weight_to_token_ratio=0.151061 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.250716465e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=358371328 dispatch_meta_bytes=229880 combine_payload_bytes=358371328 prefetch_bytes=0 reduce_bytes=660602880 egress_bytes=358601208 ingress_bytes=1018974208 link_bytes=1018974208 token_bytes=716972536 weight_movement_bytes=660602880 total_nvlink_bytes=1377575416 weight_to_token_ratio=0.921378 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=25088 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.772163740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56490 payload_rows_remote=48320 dup_rows=9046 dup_rows_remote=8892 dedup_ratio=0.138031 router_tokens_per_expert=[0, 0, 1, 0, 0, 1, 0, 3514, 0, 3539, 0, 1, 0, 3901, 1136, 20, 0, 0, 3, 3857, 0, 0, 0, 1, 1895, 0, 0, 22, 0, 19, 173, 0, 0, 0, 1406, 30, 0, 4074, 0, 3195, 0, 4092, 0, 0, 5, 0, 0, 1, 0, 0, 6, 0, 1838, 0, 5, 0, 0, 0, 1, 0, 23, 0, 9, 0] payload_rows_per_dst=[7030, 2304, 7716, 7816, 7246, 8192, 8016, 8170] dup_rows_per_dst=[2, 10, 6, 3468, 1886, 36, 3484, 154] combine_rows_per_src=[7030, 2304, 7716, 7816, 7246, 8192, 8016, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58100 payload_rows_remote=50372 dup_rows=7436 dup_rows_remote=3860 dedup_ratio=0.113464 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4075, 0, 2310, 0, 0, 0, 3995, 170, 57, 0, 0, 0, 3696, 0, 0, 0, 0, 2818, 0, 0, 40, 0, 6, 2, 0, 0, 0, 2721, 0, 0, 3985, 0, 2931, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 5, 0, 1825, 0, 0, 0, 0, 0, 32, 0, 1, 0, 0, 0] payload_rows_per_dst=[8150, 4762, 7392, 5776, 7728, 8192, 8076, 8024] dup_rows_per_dst=[0, 108, 0, 160, 3576, 0, 3580, 12] combine_rows_per_src=[8150, 4762, 7392, 5776, 7728, 8192, 8076, 8024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=295952384 dispatch_meta_bytes=186968 combine_payload_bytes=295952384 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=296139352 ingress_bytes=516153344 link_bytes=516153344 token_bytes=592091736 weight_movement_bytes=220200960 total_nvlink_bytes=812292696 weight_to_token_ratio=0.371903 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.976755579e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58210 payload_rows_remote=50114 dup_rows=7326 dup_rows_remote=2172 dedup_ratio=0.111786 router_tokens_per_expert=[0, 5, 0, 0, 1, 0, 2, 3993, 0, 2772, 0, 11, 0, 3492, 340, 10, 0, 16, 5, 3840, 1, 3, 0, 1, 3232, 0, 0, 28, 0, 1, 0, 0, 0, 0, 3024, 0, 6, 4072, 0, 1527, 0, 4081, 0, 1, 0, 1, 0, 1, 21, 4, 38, 1, 2174, 4, 2, 0, 1, 10, 10, 1, 5, 0, 29, 2] payload_rows_per_dst=[7996, 8096, 7696, 6498, 7130, 8168, 4454, 8172] dup_rows_per_dst=[6, 5154, 36, 24, 1984, 0, 34, 88] combine_rows_per_src=[7996, 8096, 7696, 6498, 7130, 8168, 4454, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=346357760 dispatch_meta_bytes=228848 combine_payload_bytes=346357760 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=346586608 ingress_bytes=566558720 link_bytes=566558720 token_bytes=692944368 weight_movement_bytes=220200960 total_nvlink_bytes=913145328 weight_to_token_ratio=0.317776 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.853387970e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56924 payload_rows_remote=48734 dup_rows=8612 dup_rows_remote=8612 dedup_ratio=0.131409 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 0, 4002, 0, 2890, 5, 2, 0, 4016, 1598, 4, 0, 0, 7, 3902, 0, 0, 0, 1, 2317, 0, 0, 8, 1, 3, 20, 0, 0, 0, 2591, 0, 0, 3963, 0, 1499, 0, 4094, 0, 0, 0, 0, 1, 0, 10, 0, 2, 0, 1821, 1, 0, 3, 0, 0, 3, 0, 1, 0, 0, 1] payload_rows_per_dst=[8008, 3208, 7816, 7380, 6256, 8190, 8132, 7934] dup_rows_per_dst=[0, 10, 4, 3098, 1924, 0, 3574, 2] combine_rows_per_src=[8008, 3208, 7816, 7380, 6256, 8190, 8132, 7934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=361066496 dispatch_meta_bytes=216928 combine_payload_bytes=361066496 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=361283424 ingress_bytes=1021669376 link_bytes=1021669376 token_bytes=722349920 weight_movement_bytes=660602880 total_nvlink_bytes=1382952800 weight_to_token_ratio=0.914519 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.776851081e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55136 payload_rows_remote=47510 dup_rows=10400 dup_rows_remote=8028 dedup_ratio=0.158691 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3738, 0, 3514, 0, 3, 0, 3598, 580, 106, 1, 0, 17, 3757, 0, 0, 0, 0, 3589, 0, 3, 1, 0, 1, 30, 0, 0, 0, 1706, 10, 1, 4089, 0, 2516, 0, 4091, 0, 0, 0, 0, 0, 0, 1, 0, 20, 0, 1379, 1, 0, 0, 0, 1, 6, 6, 2, 0, 1, 0] payload_rows_per_dst=[7476, 1360, 7528, 8106, 6680, 8182, 7626, 8178] dup_rows_per_dst=[0, 18, 22, 6170, 1786, 0, 2372, 32] combine_rows_per_src=[7476, 1360, 7528, 8106, 6680, 8182, 7626, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57210 payload_rows_remote=52340 dup_rows=8326 dup_rows_remote=8238 dedup_ratio=0.127045 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 4027, 0, 3393, 0, 0, 0, 3772, 548, 32, 0, 0, 0, 3784, 0, 0, 0, 5, 2408, 0, 0, 49, 0, 1, 19, 2, 0, 0, 2655, 1, 0, 4091, 0, 1906, 0, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 17, 0, 1941, 3, 0, 0, 0, 0, 11, 1, 4, 0, 0, 0] payload_rows_per_dst=[8056, 7364, 7570, 4870, 6652, 8190, 6316, 8192] dup_rows_per_dst=[0, 3582, 8, 88, 2472, 0, 2154, 22] combine_rows_per_src=[8056, 7364, 7570, 4870, 6652, 8190, 6316, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56372 payload_rows_remote=48614 dup_rows=9164 dup_rows_remote=9108 dedup_ratio=0.139832 router_tokens_per_expert=[0, 4, 0, 0, 4, 4, 0, 4035, 0, 3303, 0, 0, 0, 3989, 342, 19, 0, 0, 26, 3876, 1, 0, 4, 0, 2803, 0, 1, 92, 0, 0, 5, 0, 0, 0, 1836, 0, 0, 4089, 0, 2450, 0, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 6, 0, 1762, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 16] payload_rows_per_dst=[8070, 8180, 7758, 5714, 6734, 8190, 3536, 8190] dup_rows_per_dst=[24, 7126, 56, 88, 1838, 4, 0, 28] combine_rows_per_src=[8070, 8180, 7758, 5714, 6734, 8190, 3536, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=359217152 dispatch_meta_bytes=209144 combine_payload_bytes=359217152 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=359426296 ingress_bytes=689518592 link_bytes=689518592 token_bytes=718643448 weight_movement_bytes=330301440 total_nvlink_bytes=1048944888 weight_to_token_ratio=0.459618 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=14464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.199186238e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=349325312 dispatch_meta_bytes=229384 combine_payload_bytes=349325312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349554696 ingress_bytes=459425792 link_bytes=459425792 token_bytes=698880008 weight_movement_bytes=110100480 total_nvlink_bytes=808980488 weight_to_token_ratio=0.157538 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.990170148e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52372 payload_rows_remote=44288 dup_rows=13164 dup_rows_remote=10530 dedup_ratio=0.200867 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4024, 0, 2686, 0, 0, 0, 3996, 420, 25, 0, 0, 10, 3966, 0, 0, 0, 0, 3037, 0, 0, 95, 0, 2, 6, 0, 0, 0, 2534, 0, 0, 4091, 0, 1611, 0, 4096, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 2048, 0, 0, 0, 0, 0, 46, 0, 0, 0, 32, 0] payload_rows_per_dst=[8084, 8184, 8092, 6178, 1286, 8192, 4170, 8186] dup_rows_per_dst=[2634, 6070, 3836, 102, 358, 0, 12, 152] combine_rows_per_src=[8084, 8184, 8092, 6178, 1286, 8192, 4170, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=340551680 dispatch_meta_bytes=222152 combine_payload_bytes=340551680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=340773832 ingress_bytes=450652160 link_bytes=450652160 token_bytes=681325512 weight_movement_bytes=110100480 total_nvlink_bytes=791425992 weight_to_token_ratio=0.161597 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.837582258e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=375173120 dispatch_meta_bytes=242312 combine_payload_bytes=375173120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375415432 ingress_bytes=485273600 link_bytes=485273600 token_bytes=750588552 weight_movement_bytes=110100480 total_nvlink_bytes=860689032 weight_to_token_ratio=0.146686 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=22144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.439706042e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=348465152 dispatch_meta_bytes=230888 combine_payload_bytes=348465152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=348696040 ingress_bytes=458565632 link_bytes=458565632 token_bytes=697161192 weight_movement_bytes=110100480 total_nvlink_bytes=807261672 weight_to_token_ratio=0.157927 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.975210551e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61260 payload_rows_remote=53094 dup_rows=4276 dup_rows_remote=4254 dedup_ratio=0.065247 router_tokens_per_expert=[4080, 0, 0, 0, 3129, 3355, 0, 11, 2, 0, 0, 3, 4087, 16, 0, 1947, 0, 0, 2, 4079, 0, 0, 0, 0, 0, 3603, 28, 0, 0, 70, 0, 2, 0, 0, 1, 11, 0, 0, 1, 0, 2, 1, 5, 6, 0, 0, 0, 0, 0, 8, 0, 2, 3826, 3738, 3, 747, 2, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[6254, 7788, 8162, 7244, 7658, 8186, 7802, 8166] dup_rows_per_dst=[4, 2858, 0, 162, 20, 16, 1194, 22] combine_rows_per_src=[6254, 7788, 8162, 7244, 7658, 8186, 7802, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59826 payload_rows_remote=51866 dup_rows=5710 dup_rows_remote=5710 dedup_ratio=0.087128 router_tokens_per_expert=[4096, 0, 0, 0, 1718, 3601, 0, 0, 0, 32, 0, 0, 4096, 37, 0, 2198, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 3927, 0, 1050, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3436, 7960, 8186, 7866, 7960, 8192, 8034, 8192] dup_rows_per_dst=[0, 3776, 0, 0, 0, 14, 1920, 0] combine_rows_per_src=[3436, 7960, 8186, 7866, 7960, 8192, 8034, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=317456384 dispatch_meta_bytes=219272 combine_payload_bytes=317456384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=317675656 ingress_bytes=427556864 link_bytes=427556864 token_bytes=635132040 weight_movement_bytes=110100480 total_nvlink_bytes=745232520 weight_to_token_ratio=0.173351 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.435917075e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=380577792 dispatch_meta_bytes=229392 combine_payload_bytes=380577792 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=380807184 ingress_bytes=600778752 link_bytes=600778752 token_bytes=761384976 weight_movement_bytes=220200960 total_nvlink_bytes=981585936 weight_to_token_ratio=0.289211 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.044853061e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59644 payload_rows_remote=51454 dup_rows=5892 dup_rows_remote=5876 dedup_ratio=0.089905 router_tokens_per_expert=[4096, 0, 0, 0, 2185, 2594, 1, 0, 0, 4, 0, 0, 4094, 8, 0, 2677, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 3839, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 4, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 3860, 3350, 0, 1947, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[4372, 7644, 8188, 7680, 7722, 8190, 7656, 8192] dup_rows_per_dst=[0, 2922, 0, 6, 0, 16, 2944, 4] combine_rows_per_src=[4372, 7644, 8188, 7680, 7722, 8190, 7656, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=371775488 dispatch_meta_bytes=230304 combine_payload_bytes=371775488 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=372005792 ingress_bytes=591976448 link_bytes=591976448 token_bytes=743781280 weight_movement_bytes=220200960 total_nvlink_bytes=963982240 weight_to_token_ratio=0.296056 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.029544406e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58116 payload_rows_remote=53182 dup_rows=7420 dup_rows_remote=7420 dedup_ratio=0.113220 router_tokens_per_expert=[4079, 0, 0, 1, 1500, 3363, 0, 0, 0, 13, 0, 0, 4091, 7, 4, 2443, 0, 6, 1, 4079, 0, 0, 4, 5, 0, 3408, 21, 0, 0, 1, 0, 0, 0, 0, 0, 17, 2, 1, 0, 0, 7, 1, 2, 0, 0, 0, 12, 1, 0, 8, 0, 0, 4052, 3886, 0, 1751, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[7654, 4934, 8166, 7764, 8140, 8190, 5106, 8162] dup_rows_per_dst=[2074, 0, 24, 2740, 1872, 38, 672, 0] combine_rows_per_src=[7654, 4934, 8166, 7764, 8140, 8190, 5106, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59828 payload_rows_remote=52184 dup_rows=5708 dup_rows_remote=4278 dedup_ratio=0.087097 router_tokens_per_expert=[4092, 0, 0, 0, 1987, 3274, 1, 0, 1, 0, 0, 8, 4095, 14, 0, 2815, 0, 0, 0, 4087, 0, 1, 0, 3, 0, 3785, 8, 0, 3, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1, 0, 0, 0, 0, 4, 4, 0, 4019, 3457, 3, 1069, 0, 4, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[3976, 8022, 8180, 7588, 8038, 8192, 7644, 8188] dup_rows_per_dst=[0, 4202, 2, 22, 4, 36, 1430, 12] combine_rows_per_src=[3976, 8022, 8180, 7588, 8038, 8192, 7644, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60202 payload_rows_remote=52540 dup_rows=5334 dup_rows_remote=5326 dedup_ratio=0.081390 router_tokens_per_expert=[4096, 0, 0, 0, 2248, 3718, 0, 0, 0, 7, 0, 0, 4094, 47, 0, 1741, 0, 6, 2, 4092, 0, 0, 0, 0, 0, 3831, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 3816, 1, 998, 0, 6, 0, 0, 0, 0, 1, 3] payload_rows_per_dst=[6668, 5278, 8188, 7662, 8080, 8188, 7946, 8192] dup_rows_per_dst=[1874, 1702, 12, 8, 4, 30, 1684, 20] combine_rows_per_src=[6668, 5278, 8188, 7662, 8080, 8188, 7946, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=368822272 dispatch_meta_bytes=229320 combine_payload_bytes=368822272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369051592 ingress_bytes=478922752 link_bytes=478922752 token_bytes=737873864 weight_movement_bytes=110100480 total_nvlink_bytes=847974344 weight_to_token_ratio=0.149213 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.329254350e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60258 payload_rows_remote=52074 dup_rows=5278 dup_rows_remote=5274 dedup_ratio=0.080536 router_tokens_per_expert=[4087, 0, 0, 0, 2911, 3212, 1, 0, 0, 0, 0, 2, 4094, 15, 0, 2161, 0, 1, 0, 4091, 0, 0, 2, 0, 7, 3847, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 3957, 3717, 0, 641, 0, 9, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8024, 4348, 8184, 7702, 7914, 8192, 7712, 8182] dup_rows_per_dst=[4224, 8, 4, 16, 2, 10, 1004, 10] combine_rows_per_src=[8024, 4348, 8184, 7702, 7914, 8192, 7712, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=381208576 dispatch_meta_bytes=242408 combine_payload_bytes=381208576 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=381450984 ingress_bytes=601409536 link_bytes=601409536 token_bytes=762659560 weight_movement_bytes=220200960 total_nvlink_bytes=982860520 weight_to_token_ratio=0.288728 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.045950098e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54798 payload_rows_remote=46690 dup_rows=10738 dup_rows_remote=1870 dedup_ratio=0.163849 router_tokens_per_expert=[4096, 0, 0, 0, 1977, 3558, 0, 0, 0, 0, 0, 0, 4096, 65, 0, 1907, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4061, 3993, 0, 1393, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8108, 4034, 8192, 7318, 8122, 8046, 2786, 8192] dup_rows_per_dst=[8868, 60, 48, 1758, 0, 0, 2, 2] combine_rows_per_src=[8108, 4034, 8192, 7318, 8122, 8046, 2786, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=374054912 dispatch_meta_bytes=225848 combine_payload_bytes=374054912 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=374280760 ingress_bytes=924557312 link_bytes=924557312 token_bytes=748335672 weight_movement_bytes=550502400 total_nvlink_bytes=1298838072 weight_to_token_ratio=0.735636 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=23168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.607957229e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56244 payload_rows_remote=48060 dup_rows=9292 dup_rows_remote=9150 dedup_ratio=0.141785 router_tokens_per_expert=[0, 3160, 3, 16, 0, 0, 0, 0, 4074, 4, 0, 0, 3, 4086, 0, 5, 0, 0, 1269, 0, 0, 4095, 0, 0, 3, 0, 20, 0, 429, 1, 2, 0, 0, 4089, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3791, 0, 5, 3533, 0, 11, 0, 2, 0, 1, 1, 0, 0, 72, 0, 0, 4084, 0, 6, 0, 1] payload_rows_per_dst=[6338, 8088, 8190, 904, 8180, 8186, 8174, 8184] dup_rows_per_dst=[20, 14, 2538, 6, 2, 6472, 98, 142] combine_rows_per_src=[6338, 8088, 8190, 904, 8180, 8186, 8174, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=376606720 dispatch_meta_bytes=231464 combine_payload_bytes=376606720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=376838184 ingress_bytes=486707200 link_bytes=486707200 token_bytes=753444904 weight_movement_bytes=110100480 total_nvlink_bytes=863545384 weight_to_token_ratio=0.146129 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.464638704e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=373266432 dispatch_meta_bytes=229392 combine_payload_bytes=373266432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373495824 ingress_bytes=483366912 link_bytes=483366912 token_bytes=746762256 weight_movement_bytes=110100480 total_nvlink_bytes=856862736 weight_to_token_ratio=0.147437 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.406545602e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59780 payload_rows_remote=51588 dup_rows=5756 dup_rows_remote=5756 dedup_ratio=0.087830 router_tokens_per_expert=[0, 2853, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 1397, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 1481, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2677, 0, 0, 3806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[5706, 8046, 8192, 7906, 8192, 5354, 8192, 8192] dup_rows_per_dst=[0, 0, 2794, 2684, 0, 0, 0, 278] combine_rows_per_src=[5706, 8046, 8192, 7906, 8192, 5354, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=334673920 dispatch_meta_bytes=194240 combine_payload_bytes=334673920 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=334868160 ingress_bytes=775075840 link_bytes=775075840 token_bytes=669542080 weight_movement_bytes=440401920 total_nvlink_bytes=1109944000 weight_to_token_ratio=0.657766 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.347984364e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59882 payload_rows_remote=52194 dup_rows=5654 dup_rows_remote=5654 dedup_ratio=0.086273 router_tokens_per_expert=[0, 1839, 1, 0, 1, 0, 0, 0, 4094, 3, 0, 0, 0, 4044, 0, 0, 0, 0, 2003, 0, 0, 4093, 1, 0, 1, 0, 29, 0, 828, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 2, 0, 3693, 0, 0, 2, 0, 3, 0, 0, 0, 0, 98, 0, 0, 4094, 0, 2, 0, 0] payload_rows_per_dst=[3678, 8094, 8188, 7660, 8190, 7688, 8192, 8192] dup_rows_per_dst=[4, 0, 4006, 1442, 0, 0, 6, 196] combine_rows_per_src=[3678, 8094, 8188, 7660, 8190, 7688, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=344494080 dispatch_meta_bytes=228840 combine_payload_bytes=344494080 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=344722920 ingress_bytes=784896000 link_bytes=784896000 token_bytes=689217000 weight_movement_bytes=440401920 total_nvlink_bytes=1129618920 weight_to_token_ratio=0.638989 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.365063237e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=369782784 dispatch_meta_bytes=229376 combine_payload_bytes=369782784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370012160 ingress_bytes=479883264 link_bytes=479883264 token_bytes=739794944 weight_movement_bytes=110100480 total_nvlink_bytes=849895424 weight_to_token_ratio=0.148826 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.345959233e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58248 payload_rows_remote=50128 dup_rows=7288 dup_rows_remote=7286 dedup_ratio=0.111206 router_tokens_per_expert=[0, 2927, 0, 0, 1, 0, 0, 2, 4095, 0, 0, 2, 1, 4058, 0, 0, 4, 0, 1717, 0, 0, 4077, 10, 5, 0, 1, 0, 0, 687, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 2, 0, 0, 4, 3118, 1, 0, 3775, 0, 0, 0, 1, 0, 0, 0, 1, 0, 80, 0, 8, 4095, 4, 0, 1, 0] payload_rows_per_dst=[8176, 8120, 3460, 7680, 8186, 6244, 8190, 8192] dup_rows_per_dst=[5838, 2, 12, 1246, 0, 2, 4, 184] combine_rows_per_src=[8176, 8120, 3460, 7680, 8186, 6244, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60546 payload_rows_remote=52358 dup_rows=4990 dup_rows_remote=4990 dedup_ratio=0.076141 router_tokens_per_expert=[0, 2606, 1, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 1173, 0, 0, 4095, 2, 0, 0, 0, 63, 0, 643, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 5, 0, 3901, 0, 5, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 4096, 0, 2, 0, 0] payload_rows_per_dst=[5214, 8134, 8192, 6580, 8186, 7860, 8188, 8192] dup_rows_per_dst=[0, 0, 2348, 1064, 0, 1454, 0, 124] combine_rows_per_src=[5214, 8134, 8192, 6580, 8186, 7860, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=374126592 dispatch_meta_bytes=231392 combine_payload_bytes=374126592 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=374357984 ingress_bytes=594327552 link_bytes=594327552 token_bytes=748484576 weight_movement_bytes=220200960 total_nvlink_bytes=968685536 weight_to_token_ratio=0.294196 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=28160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.033633363e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61810 payload_rows_remote=53726 dup_rows=3726 dup_rows_remote=1836 dedup_ratio=0.056854 router_tokens_per_expert=[0, 2863, 0, 0, 0, 1, 0, 0, 4069, 4, 0, 0, 0, 4047, 0, 0, 0, 0, 834, 0, 0, 4095, 0, 0, 0, 0, 96, 0, 911, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 1, 2, 3980, 0, 0, 0, 1, 0, 0, 0, 0, 0, 78, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[5728, 8096, 8190, 8084, 8192, 7188, 8140, 8192] dup_rows_per_dst=[0, 6, 1668, 1890, 6, 0, 0, 156] combine_rows_per_src=[5728, 8096, 8190, 8084, 8192, 7188, 8140, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60026 payload_rows_remote=55830 dup_rows=5510 dup_rows_remote=5000 dedup_ratio=0.084076 router_tokens_per_expert=[0, 3006, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 1138, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 361, 3, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 2, 0, 3795, 0, 1, 0, 0, 0, 1, 2, 0, 0, 147, 0, 0, 4096, 0, 2, 0, 0] payload_rows_per_dst=[7722, 8180, 4196, 7682, 8192, 7704, 8158, 8192] dup_rows_per_dst=[4052, 2, 510, 642, 2, 2, 2, 298] combine_rows_per_src=[7722, 8180, 4196, 7682, 8192, 7704, 8158, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=359317504 dispatch_meta_bytes=229656 combine_payload_bytes=359317504 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=359547160 ingress_bytes=689618944 link_bytes=689618944 token_bytes=718864664 weight_movement_bytes=330301440 total_nvlink_bytes=1049166104 weight_to_token_ratio=0.459477 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.199360766e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57266 payload_rows_remote=49074 dup_rows=8270 dup_rows_remote=3402 dedup_ratio=0.126190 router_tokens_per_expert=[0, 2434, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1339, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 1089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3383, 0, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 2938, 7988, 8192, 6882, 8192, 6690] dup_rows_per_dst=[4868, 562, 114, 2078, 16, 500, 0, 132] combine_rows_per_src=[8192, 8192, 2938, 7988, 8192, 6882, 8192, 6690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47776 payload_rows_remote=39592 dup_rows=17760 dup_rows_remote=7952 dedup_ratio=0.270996 router_tokens_per_expert=[0, 0, 21, 3953, 1, 0, 3, 0, 0, 0, 4078, 0, 0, 0, 0, 608, 0, 7, 0, 0, 73, 4087, 0, 0, 0, 0, 0, 1, 7, 1, 1635, 3929, 0, 1262, 306, 0, 0, 0, 0, 3, 1, 0, 0, 0, 5, 0, 3901, 0, 0, 0, 1, 0, 0, 0, 3967, 0, 4085, 0, 815, 0, 12, 5, 1, 0] payload_rows_per_dst=[7950, 1216, 7882, 3462, 3086, 7804, 8192, 8184] dup_rows_per_dst=[6, 0, 154, 124, 56, 10, 7602, 9808] combine_rows_per_src=[7950, 1216, 7882, 3462, 3086, 7804, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=375302144 dispatch_meta_bytes=229392 combine_payload_bytes=375302144 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=375531536 ingress_bytes=595503104 link_bytes=595503104 token_bytes=750833680 weight_movement_bytes=220200960 total_nvlink_bytes=971034640 weight_to_token_ratio=0.293275 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.035677841e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=385107968 dispatch_meta_bytes=222248 combine_payload_bytes=385107968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385330216 ingress_bytes=495208448 link_bytes=495208448 token_bytes=770438184 weight_movement_bytes=110100480 total_nvlink_bytes=880538664 weight_to_token_ratio=0.142906 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=26112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.612489389e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=400189440 dispatch_meta_bytes=243320 combine_payload_bytes=400189440 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=400432760 ingress_bytes=620390400 link_bytes=620390400 token_bytes=800622200 weight_movement_bytes=220200960 total_nvlink_bytes=1020823160 weight_to_token_ratio=0.275037 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=21760 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.078960942e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57672 payload_rows_remote=50698 dup_rows=7864 dup_rows_remote=4650 dedup_ratio=0.119995 router_tokens_per_expert=[0, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1164, 0, 0, 0, 0, 217, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 697, 2511, 0, 2004, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 3454, 0, 4096, 0, 2423, 0, 3, 0, 3, 0] payload_rows_per_dst=[7896, 8192, 8190, 5788, 6974, 8002, 6908, 5722] dup_rows_per_dst=[0, 2328, 434, 628, 3214, 8, 0, 1252] combine_rows_per_src=[7896, 8192, 8190, 5788, 6974, 8002, 6908, 5722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=351762432 dispatch_meta_bytes=209904 combine_payload_bytes=351762432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351972336 ingress_bytes=461862912 link_bytes=461862912 token_bytes=703734768 weight_movement_bytes=110100480 total_nvlink_bytes=813835248 weight_to_token_ratio=0.156452 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=21760 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.032555674e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=283795456 dispatch_meta_bytes=190176 combine_payload_bytes=283795456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=283985632 ingress_bytes=503996416 link_bytes=503996416 token_bytes=567781088 weight_movement_bytes=220200960 total_nvlink_bytes=787982048 weight_to_token_ratio=0.387827 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.765326606e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57280 payload_rows_remote=49244 dup_rows=8256 dup_rows_remote=8188 dedup_ratio=0.125977 router_tokens_per_expert=[0, 3, 0, 3576, 0, 0, 1, 0, 2, 0, 4096, 0, 0, 0, 1, 806, 0, 3, 0, 0, 250, 4080, 1, 0, 0, 1, 0, 0, 0, 0, 1599, 3876, 0, 717, 506, 0, 0, 0, 0, 2, 0, 0, 0, 0, 35, 0, 4016, 1, 0, 0, 0, 0, 0, 0, 3581, 0, 4094, 0, 1520, 0, 0, 0, 0, 1] payload_rows_per_dst=[7152, 8192, 8166, 8012, 2370, 8036, 7162, 8190] dup_rows_per_dst=[8, 1618, 502, 2940, 80, 68, 0, 3040] combine_rows_per_src=[7152, 8192, 8166, 8012, 2370, 8036, 7162, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=363403264 dispatch_meta_bytes=221392 combine_payload_bytes=363403264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=363624656 ingress_bytes=473503744 link_bytes=473503744 token_bytes=727027920 weight_movement_bytes=110100480 total_nvlink_bytes=837128400 weight_to_token_ratio=0.151439 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.235008888e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57972 payload_rows_remote=49784 dup_rows=7564 dup_rows_remote=6582 dedup_ratio=0.115417 router_tokens_per_expert=[0, 1, 4, 3887, 2, 6, 0, 4, 3, 0, 4086, 0, 0, 5, 0, 491, 0, 0, 0, 0, 345, 4089, 0, 15, 0, 0, 0, 1, 0, 3, 1111, 3537, 0, 1608, 499, 0, 0, 3, 1, 11, 3, 0, 0, 9, 1, 0, 3803, 0, 0, 1, 0, 1, 0, 0, 3086, 0, 4066, 1, 2075, 0, 0, 8, 2, 0] payload_rows_per_dst=[7798, 8188, 8190, 7650, 8182, 7628, 6176, 4160] dup_rows_per_dst=[10, 982, 708, 1654, 4194, 4, 0, 12] combine_rows_per_src=[7798, 8188, 8190, 7650, 8182, 7628, 6176, 4160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54276 payload_rows_remote=46380 dup_rows=11260 dup_rows_remote=10568 dedup_ratio=0.171814 router_tokens_per_expert=[0, 0, 0, 3819, 2, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 483, 0, 0, 1, 1, 88, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 992, 3808, 0, 1859, 114, 2, 0, 0, 0, 3, 0, 0, 0, 0, 4, 0, 4072, 1, 0, 0, 0, 8, 0, 0, 3940, 0, 4078, 0, 1308, 0, 10, 5, 0, 1] payload_rows_per_dst=[7642, 2858, 8190, 7436, 3924, 8152, 7896, 8178] dup_rows_per_dst=[0, 222, 178, 1472, 32, 1762, 692, 6902] combine_rows_per_src=[7642, 2858, 8190, 7436, 3924, 8152, 7896, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58680 payload_rows_remote=51534 dup_rows=6856 dup_rows_remote=5754 dedup_ratio=0.104614 router_tokens_per_expert=[0, 0, 1, 3934, 0, 0, 0, 0, 0, 2, 4095, 1, 0, 0, 0, 968, 0, 0, 0, 0, 232, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 830, 3294, 0, 1488, 166, 0, 2, 0, 0, 0, 0, 0, 0, 0, 14, 0, 3909, 0, 0, 0, 0, 2, 0, 0, 2848, 1, 4095, 1, 2750, 0, 34, 0, 8, 1] payload_rows_per_dst=[7870, 8192, 8184, 7146, 8190, 7824, 5696, 5578] dup_rows_per_dst=[0, 1940, 464, 1102, 3312, 22, 6, 10] combine_rows_per_src=[7870, 8192, 8184, 7146, 8190, 7824, 5696, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=352980992 dispatch_meta_bytes=229728 combine_payload_bytes=352980992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=353210720 ingress_bytes=463081472 link_bytes=463081472 token_bytes=706191712 weight_movement_bytes=110100480 total_nvlink_bytes=816292192 weight_to_token_ratio=0.155907 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.053748436e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57274 payload_rows_remote=49102 dup_rows=8262 dup_rows_remote=7708 dedup_ratio=0.126068 router_tokens_per_expert=[0, 20, 0, 3943, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 808, 0, 1, 0, 0, 276, 4086, 0, 0, 0, 0, 0, 0, 6, 0, 1156, 3430, 0, 1861, 294, 0, 2, 0, 0, 50, 0, 2, 0, 0, 6, 0, 3969, 0, 0, 0, 0, 1, 0, 0, 3714, 1, 4095, 1, 919, 0, 29, 1, 0, 0] payload_rows_per_dst=[7920, 8190, 8172, 7536, 8190, 7944, 7428, 1894] dup_rows_per_dst=[10, 1616, 554, 1648, 4414, 10, 4, 6] combine_rows_per_src=[7920, 8190, 8172, 7536, 8190, 7944, 7428, 1894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=356851712 dispatch_meta_bytes=225464 combine_payload_bytes=356851712 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=357077176 ingress_bytes=687153152 link_bytes=687153152 token_bytes=713928888 weight_movement_bytes=330301440 total_nvlink_bytes=1044230328 weight_to_token_ratio=0.462653 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=25728 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.195072349e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55746 payload_rows_remote=47738 dup_rows=9790 dup_rows_remote=6524 dedup_ratio=0.149384 router_tokens_per_expert=[0, 1, 0, 3936, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1740, 0, 0, 0, 0, 454, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 813, 3656, 0, 529, 30, 0, 0, 0, 0, 4, 0, 0, 0, 0, 46, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 4096, 0, 1286, 0, 14, 0, 0, 0] payload_rows_per_dst=[8008, 8192, 5090, 7720, 8192, 8170, 7774, 2600] dup_rows_per_dst=[3266, 3610, 478, 1218, 1126, 92, 0, 0] combine_rows_per_src=[8008, 8192, 5090, 7720, 8192, 8170, 7774, 2600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46694 payload_rows_remote=40210 dup_rows=18842 dup_rows_remote=17614 dedup_ratio=0.287506 router_tokens_per_expert=[499, 0, 0, 0, 2, 4048, 0, 0, 0, 4076, 0, 0, 7, 0, 0, 0, 0, 0, 1198, 0, 0, 0, 13, 1, 185, 0, 241, 0, 0, 0, 0, 29, 3, 1685, 2606, 4092, 1, 4095, 0, 638, 1, 0, 1968, 0, 993, 2, 1, 4095, 0, 8, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 2171] payload_rows_per_dst=[2050, 8166, 2402, 3308, 7902, 8192, 8190, 6484] dup_rows_per_dst=[154, 122, 22, 742, 7524, 8814, 236, 1228] combine_rows_per_src=[2050, 8166, 2402, 3308, 7902, 8192, 8190, 6484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=332451840 dispatch_meta_bytes=227792 combine_payload_bytes=332451840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=332679632 ingress_bytes=442552320 link_bytes=442552320 token_bytes=665131472 weight_movement_bytes=110100480 total_nvlink_bytes=775231952 weight_to_token_ratio=0.165532 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=28544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.696712718e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=369395712 dispatch_meta_bytes=229152 combine_payload_bytes=369395712 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=369624864 ingress_bytes=589596672 link_bytes=589596672 token_bytes=739020576 weight_movement_bytes=220200960 total_nvlink_bytes=959221536 weight_to_token_ratio=0.297963 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=23808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.025405585e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56842 payload_rows_remote=52618 dup_rows=8694 dup_rows_remote=8630 dedup_ratio=0.132660 router_tokens_per_expert=[865, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 758, 0, 0, 0, 0, 0, 67, 0, 193, 0, 0, 0, 0, 0, 0, 3530, 2036, 4089, 0, 4096, 0, 108, 0, 0, 1704, 0, 170, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 2759] payload_rows_per_dst=[8192, 8192, 8192, 8182, 4224, 3664, 8192, 8004] dup_rows_per_dst=[1730, 0, 1516, 516, 64, 84, 210, 4574] combine_rows_per_src=[8192, 8192, 8192, 8182, 4224, 3664, 8192, 8004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=351963136 dispatch_meta_bytes=227240 combine_payload_bytes=351963136 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=352190376 ingress_bytes=682264576 link_bytes=682264576 token_bytes=704153512 weight_movement_bytes=330301440 total_nvlink_bytes=1034454952 weight_to_token_ratio=0.469076 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.186570311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=342185984 dispatch_meta_bytes=217048 combine_payload_bytes=342185984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=342403032 ingress_bytes=452286464 link_bytes=452286464 token_bytes=684589016 weight_movement_bytes=110100480 total_nvlink_bytes=794689496 weight_to_token_ratio=0.160827 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.866005492e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=288225280 dispatch_meta_bytes=231296 combine_payload_bytes=288225280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=288456576 ingress_bytes=398325760 link_bytes=398325760 token_bytes=576681856 weight_movement_bytes=110100480 total_nvlink_bytes=686782336 weight_to_token_ratio=0.190921 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=19712 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.927540100e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55560 payload_rows_remote=48146 dup_rows=9976 dup_rows_remote=4858 dedup_ratio=0.152222 router_tokens_per_expert=[290, 0, 0, 2, 3, 4095, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 1, 0, 0, 1564, 0, 0, 0, 0, 0, 231, 0, 233, 0, 0, 0, 0, 9, 6, 3122, 1854, 4095, 0, 4096, 0, 113, 0, 1, 2452, 0, 419, 0, 0, 4095, 0, 0, 1, 0, 1, 0, 105, 0, 3, 0, 0, 0, 0, 0, 0, 1884] payload_rows_per_dst=[8190, 8186, 4104, 8190, 3856, 7414, 8192, 7428] dup_rows_per_dst=[590, 2, 426, 946, 90, 5118, 214, 2590] combine_rows_per_src=[8190, 8186, 4104, 8190, 3856, 7414, 8192, 7428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=377165824 dispatch_meta_bytes=244992 combine_payload_bytes=377165824 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=377410816 ingress_bytes=817567744 link_bytes=817567744 token_bytes=754576640 weight_movement_bytes=440401920 total_nvlink_bytes=1194978560 weight_to_token_ratio=0.583641 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=18048 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.421884774e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57962 payload_rows_remote=49774 dup_rows=7574 dup_rows_remote=7560 dedup_ratio=0.115570 router_tokens_per_expert=[527, 1, 2, 0, 0, 4087, 0, 0, 2, 4092, 2, 0, 1, 1, 2, 1, 0, 0, 1228, 0, 0, 0, 0, 0, 402, 0, 103, 0, 0, 0, 0, 0, 0, 2027, 2738, 4086, 0, 4086, 0, 404, 0, 0, 1839, 1, 116, 6, 0, 4093, 0, 1, 3, 1, 0, 0, 73, 0, 14, 0, 0, 0, 0, 1, 3, 2825] payload_rows_per_dst=[8178, 8188, 8186, 8188, 7474, 3888, 8184, 5676] dup_rows_per_dst=[1056, 14, 2456, 994, 2864, 36, 144, 10] combine_rows_per_src=[8178, 8188, 8186, 8188, 7474, 3888, 8184, 5676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6291732200 prefetch_bytes=1101004800 reduce_bytes=990904320 weight_movement_bytes=2091909120 egress_bytes=3146958568 ingress_bytes=5236682752 link_bytes=5236682752 total_nvlink_bytes=8383641320 weight_to_token_ratio=0.332485 prefetch_share_of_ingress=0.210249 dedup_ratio=0.215799 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=9 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037500 gemm_inner_pad_upper_bound=11430 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.107452592e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56224 payload_rows_remote=48032 dup_rows=9312 dup_rows_remote=9074 dedup_ratio=0.142090 router_tokens_per_expert=[237, 1, 0, 0, 0, 4092, 1, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 1799, 0, 0, 0, 4, 11, 65, 0, 107, 0, 0, 0, 0, 0, 0, 2996, 1802, 4079, 0, 4096, 0, 357, 0, 0, 2412, 1, 370, 1, 2, 4092, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 2030] payload_rows_per_dst=[8192, 8186, 3614, 8188, 4202, 8188, 8192, 7462] dup_rows_per_dst=[470, 2, 14, 314, 116, 5568, 238, 2590] combine_rows_per_src=[8192, 8186, 3614, 8188, 4202, 8188, 8192, 7462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=345110528 dispatch_meta_bytes=212016 combine_payload_bytes=345110528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=345322544 ingress_bytes=565311488 link_bytes=565311488 token_bytes=690433072 weight_movement_bytes=220200960 total_nvlink_bytes=910634032 weight_to_token_ratio=0.318932 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.831696554e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6996748720 prefetch_bytes=1211105280 reduce_bytes=1431306240 weight_movement_bytes=2642411520 egress_bytes=3499524528 ingress_bytes=6139635712 link_bytes=6139635712 total_nvlink_bytes=9639160240 weight_to_token_ratio=0.377663 prefetch_share_of_ingress=0.197260 dedup_ratio=0.157166 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=13 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039338 gemm_inner_pad_upper_bound=11303 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.067783630e-04 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57586 payload_rows_remote=49450 dup_rows=7950 dup_rows_remote=7632 dedup_ratio=0.121307 router_tokens_per_expert=[452, 0, 0, 2, 0, 4087, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 0, 0, 0, 1991, 0, 0, 0, 0, 0, 34, 0, 134, 0, 0, 0, 0, 17, 0, 1997, 1803, 4042, 2, 4096, 0, 674, 1, 0, 1917, 0, 232, 3, 2, 4084, 0, 13, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 3004] payload_rows_per_dst=[8182, 8180, 8182, 8136, 4978, 4172, 8192, 7564] dup_rows_per_dst=[900, 0, 3968, 318, 458, 138, 208, 1960] combine_rows_per_src=[8182, 8180, 8182, 8136, 4978, 4172, 8192, 7564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59404 payload_rows_remote=51218 dup_rows=6132 dup_rows_remote=3844 dedup_ratio=0.093567 router_tokens_per_expert=[229, 0, 0, 0, 0, 4068, 0, 0, 1, 4052, 0, 0, 0, 0, 2, 0, 0, 0, 1149, 0, 0, 1, 0, 0, 123, 0, 297, 0, 0, 0, 0, 33, 0, 1824, 2112, 4090, 0, 4095, 0, 288, 1, 0, 2554, 0, 1039, 0, 0, 4087, 0, 9, 0, 0, 0, 0, 113, 0, 0, 0, 0, 1, 0, 0, 0, 2600] payload_rows_per_dst=[8136, 8108, 8186, 8192, 6970, 6420, 8190, 5202] dup_rows_per_dst=[458, 2, 2288, 894, 1478, 768, 244, 0] combine_rows_per_src=[8136, 8108, 8186, 8192, 6970, 6420, 8190, 5202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=356780032 dispatch_meta_bytes=229336 combine_payload_bytes=356780032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=357009368 ingress_bytes=466880512 link_bytes=466880512 token_bytes=713789400 weight_movement_bytes=110100480 total_nvlink_bytes=823889880 weight_to_token_ratio=0.154248 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.119819990e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54816 payload_rows_remote=46624 dup_rows=10720 dup_rows_remote=9854 dedup_ratio=0.163574 router_tokens_per_expert=[433, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1894, 0, 0, 0, 0, 0, 79, 0, 68, 0, 0, 0, 0, 0, 0, 2034, 2978, 4067, 0, 4096, 0, 796, 0, 0, 1348, 0, 188, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 2382] payload_rows_per_dst=[8192, 8188, 8192, 8154, 7838, 2994, 6494, 4764] dup_rows_per_dst=[866, 0, 3788, 274, 5498, 78, 216, 0] combine_rows_per_src=[8192, 8188, 8192, 8154, 7838, 2994, 6494, 4764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=344293376 dispatch_meta_bytes=228424 combine_payload_bytes=344293376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=344521800 ingress_bytes=454393856 link_bytes=454393856 token_bytes=688815176 weight_movement_bytes=110100480 total_nvlink_bytes=798915656 weight_to_token_ratio=0.159840 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.902656505e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6749047256 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=3375643096 ingress_bytes=4914810880 link_bytes=4914810880 total_nvlink_bytes=8290453976 weight_to_token_ratio=0.228389 prefetch_share_of_ingress=0.201616 dedup_ratio=0.163712 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037500 gemm_inner_pad_upper_bound=11557 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.547664468e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=354457600 dispatch_meta_bytes=228328 combine_payload_bytes=354457600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=354685928 ingress_bytes=464558080 link_bytes=464558080 token_bytes=709143528 weight_movement_bytes=110100480 total_nvlink_bytes=819244008 weight_to_token_ratio=0.155258 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=30208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.079429078e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=367130624 dispatch_meta_bytes=220248 combine_payload_bytes=367130624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=367350872 ingress_bytes=477231104 link_bytes=477231104 token_bytes=734481496 weight_movement_bytes=110100480 total_nvlink_bytes=844581976 weight_to_token_ratio=0.149902 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.299833809e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6703588608 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=3352935680 ingress_bytes=5332461568 link_bytes=5332461568 total_nvlink_bytes=8685397248 weight_to_token_ratio=0.295634 prefetch_share_of_ingress=0.185825 dedup_ratio=0.176657 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.036397 gemm_inner_pad_upper_bound=11430 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.274027706e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=334200832 dispatch_meta_bytes=225912 combine_payload_bytes=334200832 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=334426744 ingress_bytes=664502272 link_bytes=664502272 token_bytes=668627576 weight_movement_bytes=330301440 total_nvlink_bytes=998929016 weight_to_token_ratio=0.493999 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.155678743e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6684517704 prefetch_bytes=990904320 reduce_bytes=1541406720 weight_movement_bytes=2532311040 egress_bytes=3343398216 ingress_bytes=5873430528 link_bytes=5873430528 total_nvlink_bytes=9216828744 weight_to_token_ratio=0.378832 prefetch_share_of_ingress=0.168710 dedup_ratio=0.178351 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=14 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037868 gemm_inner_pad_upper_bound=11303 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.021486170e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7031904776 prefetch_bytes=1101004800 reduce_bytes=220200960 weight_movement_bytes=1321205760 egress_bytes=3517104648 ingress_bytes=4836005888 link_bytes=4836005888 total_nvlink_bytes=8353110536 weight_to_token_ratio=0.187887 prefetch_share_of_ingress=0.227668 dedup_ratio=0.149167 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=2 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=10922 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.410609626e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6932303264 prefetch_bytes=880803840 reduce_bytes=1101004800 weight_movement_bytes=1981808640 egress_bytes=3467292064 ingress_bytes=5446819840 link_bytes=5446819840 total_nvlink_bytes=8914111904 weight_to_token_ratio=0.285880 prefetch_share_of_ingress=0.161710 dedup_ratio=0.149493 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=10 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038603 gemm_inner_pad_upper_bound=11049 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.472915550e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6266798072 prefetch_bytes=990904320 reduce_bytes=1431306240 weight_movement_bytes=2422210560 egress_bytes=3134439416 ingress_bytes=5554569216 link_bytes=5554569216 total_nvlink_bytes=8689008632 weight_to_token_ratio=0.386515 prefetch_share_of_ingress=0.178394 dedup_ratio=0.214017 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=13 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039338 gemm_inner_pad_upper_bound=11176 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.660309436e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none + [2026-08-26 18:09:52] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 369.7 | throughput per GPU (TFLOP/s/GPU): 334.0 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.110826E+01 | loss scale: 1.0 | grad norm: 107.484 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47552 payload_rows_remote=41120 dup_rows=17984 dup_rows_remote=15186 dedup_ratio=0.274414 router_tokens_per_expert=[193, 82, 91, 16, 6, 9, 1335, 1104, 228, 76, 161, 376, 3, 142, 0, 182, 209, 307, 2517, 11, 341, 206, 23, 164, 1950, 350, 174, 1057, 18, 1768, 609, 6, 21, 6, 324, 2157, 3, 0, 13, 141, 500, 858, 941, 23, 2542, 1303, 805, 740, 109, 1306, 763, 61, 641, 545, 303, 1010, 1516, 669, 111, 67, 303, 421, 851, 1] payload_rows_per_dst=[5294, 6148, 5980, 5926, 6432, 5836, 6342, 5594] dup_rows_per_dst=[1988, 1272, 1576, 2038, 2798, 2894, 3134, 2284] combine_rows_per_src=[5294, 6148, 5980, 5926, 6432, 5836, 6342, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47758 payload_rows_remote=40660 dup_rows=17778 dup_rows_remote=12452 dedup_ratio=0.271271 router_tokens_per_expert=[261, 40, 69, 18, 15, 48, 1329, 1275, 138, 40, 134, 392, 8, 49, 1, 127, 180, 330, 2719, 5, 131, 303, 30, 124, 1457, 225, 178, 829, 8, 1616, 657, 5, 24, 15, 69, 2591, 10, 0, 4, 141, 912, 698, 851, 64, 2704, 1467, 2457, 503, 226, 1927, 229, 102, 343, 379, 450, 571, 1201, 618, 217, 97, 426, 170, 560, 1] payload_rows_per_dst=[5248, 6190, 6220, 5418, 6342, 7098, 6126, 5116] dup_rows_per_dst=[1986, 996, 1424, 1618, 2280, 5326, 2328, 1820] combine_rows_per_src=[5248, 6190, 6220, 5418, 6342, 7098, 6126, 5116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46632 payload_rows_remote=39690 dup_rows=18904 dup_rows_remote=15932 dedup_ratio=0.288452 router_tokens_per_expert=[287, 32, 127, 47, 250, 17, 877, 2010, 225, 159, 96, 805, 12, 118, 0, 243, 97, 795, 1703, 18, 139, 222, 52, 159, 1757, 448, 193, 1253, 75, 1748, 1008, 11, 27, 16, 112, 1862, 14, 9, 15, 208, 606, 941, 642, 75, 2657, 1065, 898, 730, 376, 1326, 571, 115, 410, 428, 451, 830, 934, 743, 148, 113, 516, 288, 655, 4] payload_rows_per_dst=[6118, 6942, 6112, 6424, 5882, 3926, 6174, 5054] dup_rows_per_dst=[2972, 2972, 2388, 3048, 2158, 778, 2840, 1748] combine_rows_per_src=[6118, 6942, 6112, 6424, 5882, 3926, 6174, 5054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47798 payload_rows_remote=42272 dup_rows=17738 dup_rows_remote=16150 dedup_ratio=0.270660 router_tokens_per_expert=[155, 71, 27, 11, 70, 18, 1406, 1768, 314, 45, 140, 604, 5, 33, 0, 266, 81, 487, 2597, 28, 151, 183, 21, 157, 1429, 186, 215, 664, 6, 1726, 739, 21, 37, 24, 94, 2340, 2, 0, 57, 398, 795, 668, 902, 13, 2354, 652, 2450, 1220, 204, 1112, 561, 29, 291, 453, 209, 601, 1429, 326, 111, 209, 259, 551, 790, 3] payload_rows_per_dst=[7244, 6110, 6110, 5526, 6224, 5902, 5474, 5208] dup_rows_per_dst=[4708, 1412, 1300, 1588, 2538, 2598, 1446, 2148] combine_rows_per_src=[7244, 6110, 6110, 5526, 6224, 5902, 5474, 5208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46458 payload_rows_remote=41464 dup_rows=19078 dup_rows_remote=17718 dedup_ratio=0.291107 router_tokens_per_expert=[229, 54, 257, 57, 32, 16, 1202, 1431, 354, 67, 198, 442, 9, 115, 1, 262, 104, 734, 2017, 4, 320, 165, 37, 57, 1965, 364, 365, 1106, 9, 1923, 832, 31, 5, 27, 281, 1906, 22, 0, 17, 334, 523, 1082, 984, 43, 2404, 1027, 648, 869, 71, 908, 684, 137, 448, 300, 216, 413, 1639, 664, 197, 147, 437, 618, 955, 3] payload_rows_per_dst=[4878, 6052, 5388, 6430, 6286, 5556, 4994, 6874] dup_rows_per_dst=[1678, 1652, 1488, 2830, 2828, 2632, 1360, 4610] combine_rows_per_src=[4878, 6052, 5388, 6430, 6286, 5556, 4994, 6874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47488 payload_rows_remote=41576 dup_rows=18048 dup_rows_remote=15008 dedup_ratio=0.275391 router_tokens_per_expert=[208, 18, 382, 41, 12, 13, 1814, 897, 308, 95, 158, 271, 9, 238, 0, 188, 335, 200, 2989, 0, 229, 192, 45, 101, 1355, 842, 145, 803, 50, 1564, 786, 16, 21, 24, 388, 1890, 7, 9, 16, 85, 440, 658, 928, 53, 2356, 1152, 1427, 790, 211, 1656, 498, 108, 524, 651, 233, 521, 1472, 365, 259, 151, 330, 97, 1142, 2] payload_rows_per_dst=[5266, 5914, 6610, 5792, 5790, 5954, 6250, 5912] dup_rows_per_dst=[1504, 1332, 1572, 2134, 1800, 3626, 3040, 3040] combine_rows_per_src=[5266, 5914, 6610, 5792, 5790, 5954, 6250, 5912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 2, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=294748160 dispatch_meta_bytes=225224 combine_payload_bytes=294748160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=294973384 ingress_bytes=404848640 link_bytes=404848640 token_bytes=589721544 weight_movement_bytes=110100480 total_nvlink_bytes=699822024 weight_to_token_ratio=0.186699 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.040983711e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=291450880 dispatch_meta_bytes=212448 combine_payload_bytes=291450880 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=291663328 ingress_bytes=952053760 link_bytes=952053760 token_bytes=583114208 weight_movement_bytes=660602880 total_nvlink_bytes=1243717088 weight_to_token_ratio=1.132888 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.655778075e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284497920 dispatch_meta_bytes=222488 combine_payload_bytes=284497920 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=284720408 ingress_bytes=504698880 link_bytes=504698880 token_bytes=569218328 weight_movement_bytes=220200960 total_nvlink_bytes=789419288 weight_to_token_ratio=0.386848 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=20096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.777543610e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46816 payload_rows_remote=40412 dup_rows=18720 dup_rows_remote=15990 dedup_ratio=0.285645 router_tokens_per_expert=[278, 71, 105, 20, 41, 26, 979, 1027, 151, 158, 228, 524, 47, 165, 1, 147, 102, 565, 2425, 36, 511, 276, 14, 55, 1841, 271, 431, 884, 24, 1377, 720, 17, 27, 21, 119, 1759, 9, 6, 8, 355, 978, 951, 945, 25, 2328, 1359, 1250, 797, 250, 1320, 890, 111, 463, 612, 333, 709, 1197, 698, 192, 214, 301, 181, 843, 0] payload_rows_per_dst=[5418, 6540, 6404, 5696, 5976, 5274, 6182, 5326] dup_rows_per_dst=[2176, 2618, 2730, 1752, 2314, 2010, 3194, 1926] combine_rows_per_src=[5418, 6540, 6404, 5696, 5976, 5274, 6182, 5326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46976 payload_rows_remote=40974 dup_rows=18560 dup_rows_remote=15504 dedup_ratio=0.283203 router_tokens_per_expert=[198, 83, 104, 26, 69, 14, 1370, 1546, 334, 15, 95, 478, 24, 44, 3, 222, 248, 587, 2334, 8, 237, 247, 13, 136, 1586, 298, 253, 981, 25, 1239, 835, 35, 47, 14, 114, 2004, 10, 1, 8, 152, 733, 825, 820, 99, 2658, 1307, 1119, 815, 216, 1345, 328, 243, 580, 301, 230, 674, 1672, 1025, 237, 92, 304, 526, 574, 8] payload_rows_per_dst=[6002, 6826, 6918, 6414, 4880, 4644, 5178, 6114] dup_rows_per_dst=[3056, 2560, 3316, 2902, 1008, 1482, 1474, 2762] combine_rows_per_src=[6002, 6826, 6918, 6414, 4880, 4644, 5178, 6114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=303005696 dispatch_meta_bytes=233688 combine_payload_bytes=303005696 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=303239384 ingress_bytes=523206656 link_bytes=523206656 token_bytes=606245080 weight_movement_bytes=220200960 total_nvlink_bytes=826446040 weight_to_token_ratio=0.363221 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.099424275e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=297213952 dispatch_meta_bytes=236728 combine_payload_bytes=297213952 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=297450680 ingress_bytes=517414912 link_bytes=517414912 token_bytes=594664632 weight_movement_bytes=220200960 total_nvlink_bytes=814865592 weight_to_token_ratio=0.370294 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.998696321e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=298016768 dispatch_meta_bytes=226336 combine_payload_bytes=298016768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298243104 ingress_bytes=408117248 link_bytes=408117248 token_bytes=596259872 weight_movement_bytes=110100480 total_nvlink_bytes=706360352 weight_to_token_ratio=0.184652 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.097830180e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=289673216 dispatch_meta_bytes=225608 combine_payload_bytes=289673216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=289898824 ingress_bytes=399773696 link_bytes=399773696 token_bytes=579572040 weight_movement_bytes=110100480 total_nvlink_bytes=689672520 weight_to_token_ratio=0.189969 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.952722088e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=293701632 dispatch_meta_bytes=225912 combine_payload_bytes=293701632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293927544 ingress_bytes=403802112 link_bytes=403802112 token_bytes=587629176 weight_movement_bytes=110100480 total_nvlink_bytes=697729656 weight_to_token_ratio=0.187364 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.022782868e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53126 payload_rows_remote=44948 dup_rows=12410 dup_rows_remote=12304 dedup_ratio=0.189362 router_tokens_per_expert=[0, 4084, 2760, 387, 4, 379, 1427, 2158, 0, 760, 22, 0, 0, 0, 2, 0, 0, 1277, 838, 24, 0, 35, 0, 2, 26, 4042, 7, 0, 2, 0, 2821, 0, 0, 52, 0, 2, 0, 0, 0, 4, 262, 0, 2665, 3, 13, 0, 0, 0, 0, 0, 1, 80, 0, 166, 0, 0, 1375, 3, 124, 0, 0, 1, 3781, 3179] payload_rows_per_dst=[6426, 6000, 3788, 8146, 8178, 5552, 7772, 7264] dup_rows_per_dst=[2284, 1088, 564, 5650, 106, 334, 284, 2100] combine_rows_per_src=[6426, 6000, 3788, 8146, 8178, 5552, 7772, 7264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53158 payload_rows_remote=46382 dup_rows=12378 dup_rows_remote=11450 dedup_ratio=0.188873 router_tokens_per_expert=[0, 4080, 2338, 494, 0, 130, 1800, 2046, 1, 1696, 20, 0, 0, 0, 7, 1, 0, 748, 548, 18, 0, 54, 2, 2, 55, 4052, 3, 3, 0, 2, 2270, 0, 1, 78, 1, 0, 0, 0, 0, 0, 671, 0, 3175, 3, 1, 0, 2, 0, 0, 0, 0, 127, 12, 97, 4, 5, 1113, 0, 62, 2, 3, 7, 3608, 3426] payload_rows_per_dst=[6374, 6446, 2462, 8158, 8172, 6776, 7348, 7422] dup_rows_per_dst=[2460, 1786, 282, 4612, 148, 928, 358, 1804] combine_rows_per_src=[6374, 6446, 2462, 8158, 8172, 6776, 7348, 7422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53610 payload_rows_remote=47398 dup_rows=11926 dup_rows_remote=10426 dedup_ratio=0.181976 router_tokens_per_expert=[0, 4082, 2792, 458, 2, 274, 1580, 2259, 0, 1042, 16, 2, 1, 1, 0, 2, 0, 565, 1762, 6, 0, 63, 10, 10, 128, 3743, 32, 0, 1, 3, 1978, 0, 0, 3, 0, 1, 1, 0, 0, 0, 563, 1, 2760, 4, 3, 0, 0, 0, 6, 0, 2, 75, 19, 138, 0, 0, 1612, 0, 81, 4, 0, 0, 3802, 2881] payload_rows_per_dst=[6554, 6212, 7904, 4164, 8168, 5842, 7760, 7006] dup_rows_per_dst=[2592, 1500, 4414, 120, 6, 820, 324, 2150] combine_rows_per_src=[6554, 6212, 7904, 4164, 8168, 5842, 7760, 7006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54132 payload_rows_remote=47526 dup_rows=11404 dup_rows_remote=10038 dedup_ratio=0.174011 router_tokens_per_expert=[0, 4095, 2254, 294, 0, 471, 1472, 2159, 0, 776, 0, 1, 0, 0, 0, 42, 0, 1097, 846, 8, 0, 27, 3, 4, 33, 3929, 33, 0, 0, 0, 2795, 0, 13, 51, 0, 3, 0, 0, 0, 0, 423, 0, 2739, 0, 0, 3, 4, 0, 0, 24, 1, 200, 7, 317, 1, 0, 1778, 0, 133, 0, 0, 14, 3817, 2901] payload_rows_per_dst=[6446, 5248, 6820, 6606, 8190, 5806, 7854, 7162] dup_rows_per_dst=[2346, 898, 2758, 1366, 134, 532, 880, 2490] combine_rows_per_src=[6446, 5248, 6820, 6606, 8190, 5806, 7854, 7162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52806 payload_rows_remote=44994 dup_rows=12730 dup_rows_remote=11774 dedup_ratio=0.194244 router_tokens_per_expert=[0, 4094, 2826, 362, 0, 199, 1415, 1373, 0, 1046, 23, 0, 0, 0, 0, 0, 0, 1567, 1043, 2, 0, 46, 0, 3, 69, 4020, 18, 0, 0, 0, 2693, 0, 0, 116, 0, 0, 0, 0, 0, 0, 660, 0, 3452, 0, 2, 0, 0, 0, 0, 0, 0, 123, 0, 534, 0, 0, 987, 0, 126, 0, 0, 0, 3727, 2242] payload_rows_per_dst=[3538, 7118, 4456, 8074, 8188, 7824, 7812, 5796] dup_rows_per_dst=[414, 3418, 866, 1232, 232, 4698, 956, 914] combine_rows_per_src=[3538, 7118, 4456, 8074, 8188, 7824, 7812, 5796] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53122 payload_rows_remote=45886 dup_rows=12414 dup_rows_remote=10108 dedup_ratio=0.189423 router_tokens_per_expert=[0, 4094, 3405, 388, 0, 221, 1676, 1577, 0, 835, 2, 0, 3, 0, 0, 0, 0, 634, 1021, 7, 1, 55, 0, 4, 33, 4001, 2, 0, 11, 0, 2034, 1, 1, 54, 0, 0, 0, 0, 0, 0, 667, 0, 3369, 0, 1, 1, 0, 0, 1, 0, 0, 75, 1, 438, 0, 0, 1505, 0, 166, 0, 0, 0, 3407, 3077] payload_rows_per_dst=[4080, 7570, 3108, 8008, 8190, 7788, 7142, 7236] dup_rows_per_dst=[490, 4074, 336, 88, 108, 4356, 656, 2306] combine_rows_per_src=[4080, 7570, 3108, 8008, 8190, 7788, 7142, 7236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=322187264 dispatch_meta_bytes=229008 combine_payload_bytes=322187264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=322416272 ingress_bytes=432287744 link_bytes=432287744 token_bytes=644603536 weight_movement_bytes=110100480 total_nvlink_bytes=754704016 weight_to_token_ratio=0.170803 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.518194859e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=332466176 dispatch_meta_bytes=231328 combine_payload_bytes=332466176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=332697504 ingress_bytes=442566656 link_bytes=442566656 token_bytes=665163680 weight_movement_bytes=110100480 total_nvlink_bytes=775264160 weight_to_token_ratio=0.165524 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.696962045e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=339748864 dispatch_meta_bytes=231296 combine_payload_bytes=339748864 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=339980160 ingress_bytes=559949824 link_bytes=559949824 token_bytes=679729024 weight_movement_bytes=220200960 total_nvlink_bytes=899929984 weight_to_token_ratio=0.323954 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.033088 gemm_active_groups=11 gemm_peak_group_rows=22016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.738448399e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=340666368 dispatch_meta_bytes=230256 combine_payload_bytes=340666368 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=340896624 ingress_bytes=670967808 link_bytes=670967808 token_bytes=681562992 weight_movement_bytes=330301440 total_nvlink_bytes=1011864432 weight_to_token_ratio=0.484623 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.166923373e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=322516992 dispatch_meta_bytes=227072 combine_payload_bytes=322516992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=322744064 ingress_bytes=432617472 link_bytes=432617472 token_bytes=645261056 weight_movement_bytes=110100480 total_nvlink_bytes=755361536 weight_to_token_ratio=0.170629 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.523929372e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=328910848 dispatch_meta_bytes=223976 combine_payload_bytes=328910848 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=329134824 ingress_bytes=659212288 link_bytes=659212288 token_bytes=658045672 weight_movement_bytes=330301440 total_nvlink_bytes=988347112 weight_to_token_ratio=0.501943 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.146478591e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52838 payload_rows_remote=44664 dup_rows=12698 dup_rows_remote=5862 dedup_ratio=0.193756 router_tokens_per_expert=[0, 4085, 2846, 367, 0, 119, 1116, 1948, 0, 687, 7, 0, 0, 0, 0, 0, 0, 1261, 2070, 18, 0, 95, 0, 0, 54, 4061, 111, 0, 0, 0, 1925, 0, 0, 86, 0, 0, 0, 0, 0, 0, 388, 0, 3166, 0, 1, 0, 0, 0, 0, 0, 0, 439, 0, 297, 0, 0, 1121, 0, 138, 0, 0, 0, 3362, 3000] payload_rows_per_dst=[5650, 6114, 8174, 4060, 8170, 6600, 7226, 6844] dup_rows_per_dst=[1450, 966, 6836, 120, 172, 510, 970, 1674] combine_rows_per_src=[5650, 6114, 8174, 4060, 8170, 6600, 7226, 6844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50726 payload_rows_remote=43190 dup_rows=14810 dup_rows_remote=7914 dedup_ratio=0.225983 router_tokens_per_expert=[0, 4095, 2698, 283, 4, 342, 2007, 1807, 0, 832, 19, 0, 0, 0, 0, 0, 0, 1110, 1246, 52, 0, 67, 0, 9, 23, 3900, 0, 0, 0, 0, 2447, 0, 0, 103, 0, 0, 0, 0, 0, 0, 747, 0, 2837, 0, 0, 0, 0, 1, 0, 0, 0, 87, 1, 234, 0, 0, 997, 1, 103, 0, 0, 0, 3775, 2941] payload_rows_per_dst=[7536, 5996, 8024, 4972, 7190, 6268, 7734, 3006] dup_rows_per_dst=[6896, 1102, 4744, 160, 184, 902, 460, 362] combine_rows_per_src=[7536, 5996, 8024, 4972, 7190, 6268, 7734, 3006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50840 payload_rows_remote=43490 dup_rows=14696 dup_rows_remote=11028 dedup_ratio=0.224243 router_tokens_per_expert=[2163, 3212, 1999, 8, 2457, 0, 0, 0, 0, 147, 0, 423, 43, 2, 1466, 0, 0, 0, 0, 1418, 1607, 0, 0, 0, 0, 4095, 0, 1445, 1, 2, 0, 0, 3282, 922, 1, 245, 0, 0, 7, 1252, 0, 0, 2, 0, 4095, 0, 872, 1, 0, 0, 3, 0, 7, 0, 0, 0, 0, 0, 1564, 2, 0, 0, 1, 24] payload_rows_per_dst=[5510, 4322, 5152, 7004, 7350, 8190, 7266, 6046] dup_rows_per_dst=[1120, 1324, 1298, 2598, 3668, 1750, 1804, 1134] combine_rows_per_src=[5510, 4322, 5152, 7004, 7350, 8190, 7266, 6046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=320151552 dispatch_meta_bytes=202104 combine_payload_bytes=320151552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=320353656 ingress_bytes=430252032 link_bytes=430252032 token_bytes=640505208 weight_movement_bytes=110100480 total_nvlink_bytes=750605688 weight_to_token_ratio=0.171896 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=14592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.482790479e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=309585920 dispatch_meta_bytes=204416 combine_payload_bytes=309585920 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=309790336 ingress_bytes=749987840 link_bytes=749987840 token_bytes=619376256 weight_movement_bytes=440401920 total_nvlink_bytes=1059778176 weight_to_token_ratio=0.711041 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=12544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.304352206e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47198 payload_rows_remote=39006 dup_rows=18338 dup_rows_remote=14410 dedup_ratio=0.279816 router_tokens_per_expert=[2833, 3470, 2920, 0, 1548, 2, 0, 0, 7, 111, 0, 664, 29, 4, 743, 0, 0, 2, 0, 1732, 1308, 1, 1, 3, 9, 4080, 3, 2204, 0, 5, 10, 0, 2305, 393, 0, 446, 0, 1, 46, 1096, 2, 5, 8, 0, 4081, 11, 1953, 0, 0, 0, 0, 1, 4, 0, 2, 0, 3, 0, 660, 0, 0, 7, 0, 55] payload_rows_per_dst=[2386, 2816, 5028, 8186, 6186, 8192, 7808, 6596] dup_rows_per_dst=[2, 300, 1066, 4436, 2388, 3928, 4812, 1406] combine_rows_per_src=[2386, 2816, 5028, 8186, 6186, 8192, 7808, 6596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47394 payload_rows_remote=39208 dup_rows=18142 dup_rows_remote=14346 dedup_ratio=0.276825 router_tokens_per_expert=[2569, 3033, 2319, 3, 1849, 3, 0, 0, 0, 142, 0, 501, 297, 4, 957, 0, 5, 0, 0, 1767, 946, 2, 0, 3, 1, 4090, 0, 2497, 0, 14, 0, 0, 2852, 1004, 1, 678, 1, 0, 4, 1382, 0, 0, 2, 0, 4087, 0, 1060, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4, 661, 0, 0, 6, 4, 18] payload_rows_per_dst=[8190, 8186, 7456, 5004, 4956, 2124, 6066, 5412] dup_rows_per_dst=[8832, 3796, 3694, 20, 1184, 0, 4, 612] combine_rows_per_src=[8190, 8186, 7456, 5004, 4956, 2124, 6066, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=311736320 dispatch_meta_bytes=218072 combine_payload_bytes=311736320 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=311954392 ingress_bytes=642037760 link_bytes=642037760 token_bytes=623690712 weight_movement_bytes=330301440 total_nvlink_bytes=953992152 weight_to_token_ratio=0.529592 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.116609262e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47016 payload_rows_remote=39320 dup_rows=18520 dup_rows_remote=13416 dedup_ratio=0.282593 router_tokens_per_expert=[2498, 2389, 3274, 0, 2087, 0, 0, 0, 0, 72, 1, 472, 109, 8, 1340, 0, 0, 0, 0, 1200, 1785, 0, 1, 0, 0, 4096, 1, 1257, 0, 13, 0, 1, 2684, 1340, 0, 494, 0, 3, 26, 1562, 0, 1, 2, 0, 4095, 0, 915, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1019, 0, 0, 0, 0, 20] payload_rows_per_dst=[0, 3596, 5102, 8192, 7166, 8190, 7074, 7696] dup_rows_per_dst=[0, 408, 870, 2544, 5052, 1836, 2706, 5104] combine_rows_per_src=[0, 3596, 5102, 8192, 7166, 8190, 7074, 7696] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44838 payload_rows_remote=38162 dup_rows=20698 dup_rows_remote=18842 dedup_ratio=0.315826 router_tokens_per_expert=[1786, 2456, 3087, 0, 2896, 0, 0, 0, 0, 72, 0, 360, 9, 6, 1029, 0, 0, 0, 0, 880, 1445, 0, 0, 0, 0, 4096, 0, 2369, 0, 0, 0, 0, 2441, 1077, 0, 635, 0, 8, 22, 2200, 0, 0, 1, 0, 4096, 0, 885, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 881, 0, 0, 0, 0, 7] payload_rows_per_dst=[0, 2628, 4038, 8192, 7328, 8192, 6676, 7784] dup_rows_per_dst=[0, 324, 612, 4738, 5438, 1772, 1856, 5958] combine_rows_per_src=[0, 2628, 4038, 8192, 7328, 8192, 6676, 7784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=281042944 dispatch_meta_bytes=214216 combine_payload_bytes=281042944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281257160 ingress_bytes=391143424 link_bytes=391143424 token_bytes=562300104 weight_movement_bytes=110100480 total_nvlink_bytes=672400584 weight_to_token_ratio=0.195804 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.802627464e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=213664 combine_payload_bytes=279595008 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279808672 ingress_bytes=499795968 link_bytes=499795968 token_bytes=559403680 weight_movement_bytes=220200960 total_nvlink_bytes=779604640 weight_to_token_ratio=0.393635 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.692273907e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47490 payload_rows_remote=44852 dup_rows=18046 dup_rows_remote=18044 dedup_ratio=0.275360 router_tokens_per_expert=[2327, 3585, 1650, 16, 1770, 34, 0, 0, 0, 138, 0, 1831, 19, 8, 1500, 0, 3, 0, 0, 1190, 1782, 0, 0, 0, 0, 4091, 0, 1319, 0, 1, 0, 0, 2603, 155, 0, 585, 0, 0, 17, 1226, 0, 8, 0, 0, 4078, 4, 2092, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 701, 1, 0, 3, 0, 15] payload_rows_per_dst=[6082, 8184, 7220, 2638, 3614, 8182, 7172, 4398] dup_rows_per_dst=[2212, 6990, 3936, 2, 352, 4182, 30, 342] combine_rows_per_src=[6082, 8184, 7220, 2638, 3614, 8182, 7172, 4398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=281845760 dispatch_meta_bytes=210944 combine_payload_bytes=281845760 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=282056704 ingress_bytes=502046720 link_bytes=502046720 token_bytes=563902464 weight_movement_bytes=220200960 total_nvlink_bytes=784103424 weight_to_token_ratio=0.390495 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.029412 gemm_active_groups=11 gemm_peak_group_rows=20224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.731418186e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45494 payload_rows_remote=37304 dup_rows=20042 dup_rows_remote=10266 dedup_ratio=0.305817 router_tokens_per_expert=[2599, 2888, 2162, 0, 2289, 0, 0, 0, 0, 89, 0, 234, 50, 0, 1265, 0, 0, 0, 0, 1075, 1549, 0, 0, 0, 1, 4095, 0, 2361, 0, 11, 0, 0, 3255, 840, 0, 179, 0, 0, 1, 2559, 2, 0, 10, 0, 4095, 0, 675, 0, 0, 0, 0, 0, 19, 1, 0, 1, 0, 0, 379, 0, 1, 0, 0, 83] payload_rows_per_dst=[8190, 8190, 7720, 5968, 3320, 1372, 5798, 4936] dup_rows_per_dst=[9776, 3276, 4038, 2150, 466, 2, 20, 314] combine_rows_per_src=[8190, 8190, 7720, 5968, 3320, 1372, 5798, 4936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273545216 dispatch_meta_bytes=228016 combine_payload_bytes=273545216 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=273773232 ingress_bytes=493746176 link_bytes=493746176 token_bytes=547318448 weight_movement_bytes=220200960 total_nvlink_bytes=767519408 weight_to_token_ratio=0.402327 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=24320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.587058074e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50244 payload_rows_remote=43912 dup_rows=15292 dup_rows_remote=13318 dedup_ratio=0.233337 router_tokens_per_expert=[2708, 3218, 2791, 0, 2433, 0, 0, 0, 0, 97, 0, 276, 81, 2, 1046, 0, 0, 0, 0, 903, 888, 0, 0, 0, 0, 4096, 0, 1914, 0, 18, 0, 0, 2362, 847, 0, 315, 0, 3, 44, 2159, 0, 0, 0, 0, 4096, 0, 1359, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1066, 0, 0, 0, 0, 42] payload_rows_per_dst=[7420, 8192, 6332, 3846, 5562, 5876, 6436, 6580] dup_rows_per_dst=[5816, 3004, 1974, 18, 1506, 1748, 8, 1218] combine_rows_per_src=[7420, 8192, 6332, 3846, 5562, 5876, 6436, 6580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=321499136 dispatch_meta_bytes=251584 combine_payload_bytes=321499136 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=321750720 ingress_bytes=541700096 link_bytes=541700096 token_bytes=643249856 weight_movement_bytes=220200960 total_nvlink_bytes=863450816 weight_to_token_ratio=0.342326 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.421055613e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56094 payload_rows_remote=48482 dup_rows=9442 dup_rows_remote=5422 dedup_ratio=0.144073 router_tokens_per_expert=[0, 0, 0, 250, 3771, 0, 0, 0, 34, 0, 2, 0, 1, 2603, 0, 0, 0, 1388, 0, 0, 0, 2, 5, 475, 4089, 0, 1, 5, 0, 0, 0, 1213, 0, 97, 0, 0, 0, 4089, 3469, 0, 1, 0, 0, 0, 4094, 159, 0, 0, 554, 0, 0, 2216, 0, 0, 2, 158, 0, 4088, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[7576, 5260, 5676, 8184, 7612, 8190, 5418, 8178] dup_rows_per_dst=[466, 20, 1742, 2432, 4020, 318, 442, 2] combine_rows_per_src=[7576, 5260, 5676, 8184, 7612, 8190, 5418, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56018 payload_rows_remote=47842 dup_rows=9518 dup_rows_remote=6030 dedup_ratio=0.145233 router_tokens_per_expert=[0, 0, 0, 243, 3754, 0, 0, 0, 6, 0, 0, 0, 0, 1746, 0, 0, 0, 1344, 14, 6, 0, 4, 17, 500, 4080, 0, 6, 1, 1, 0, 2, 720, 0, 107, 10, 0, 1, 4087, 3801, 1, 2, 0, 1, 0, 4083, 177, 0, 0, 1442, 0, 0, 2506, 0, 0, 3, 12, 7, 4082, 1, 0, 0, 1, 0, 0] payload_rows_per_dst=[7564, 8176, 8190, 1458, 7610, 8170, 6676, 8174] dup_rows_per_dst=[430, 3488, 3754, 2, 230, 356, 1250, 8] combine_rows_per_src=[7564, 8176, 8190, 1458, 7610, 8170, 6676, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=190280 combine_payload_bytes=267395072 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=267585352 ingress_bytes=817897472 link_bytes=817897472 token_bytes=534980424 weight_movement_bytes=550502400 total_nvlink_bytes=1085482824 weight_to_token_ratio=1.029014 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.422458225e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54246 payload_rows_remote=46066 dup_rows=11290 dup_rows_remote=11014 dedup_ratio=0.172272 router_tokens_per_expert=[0, 0, 0, 285, 3824, 0, 0, 2, 9, 0, 1, 2, 0, 2591, 0, 0, 6, 552, 0, 0, 0, 2, 3, 907, 4063, 0, 0, 0, 0, 1, 19, 1454, 2, 128, 1, 0, 5, 4096, 3384, 0, 0, 0, 0, 0, 4090, 138, 0, 0, 668, 0, 0, 2388, 1, 0, 0, 37, 9, 4094, 1, 0, 1, 0, 4, 0] payload_rows_per_dst=[7674, 5198, 2874, 8168, 8192, 8180, 5768, 8192] dup_rows_per_dst=[548, 8, 66, 2906, 7040, 276, 420, 26] combine_rows_per_src=[7674, 5198, 2874, 8168, 8192, 8180, 5768, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=314761216 dispatch_meta_bytes=228920 combine_payload_bytes=314761216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=314990136 ingress_bytes=424861696 link_bytes=424861696 token_bytes=629751352 weight_movement_bytes=110100480 total_nvlink_bytes=739851832 weight_to_token_ratio=0.174832 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=11392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.389043671e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=347518976 dispatch_meta_bytes=215616 combine_payload_bytes=347518976 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=347734592 ingress_bytes=787920896 link_bytes=787920896 token_bytes=695253568 weight_movement_bytes=440401920 total_nvlink_bytes=1135655488 weight_to_token_ratio=0.633441 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.370324029e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56350 payload_rows_remote=48158 dup_rows=9186 dup_rows_remote=9180 dedup_ratio=0.140167 router_tokens_per_expert=[0, 1, 0, 156, 3292, 1, 0, 0, 5, 0, 0, 0, 0, 2481, 0, 0, 0, 1241, 0, 0, 0, 0, 0, 885, 4088, 1, 6, 0, 0, 0, 0, 1072, 1, 68, 0, 1, 0, 4094, 3410, 1, 0, 0, 0, 0, 4095, 121, 0, 0, 1152, 0, 0, 2493, 0, 0, 0, 4, 2, 4095, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[6604, 4972, 3950, 8182, 8192, 8190, 8068, 8192] dup_rows_per_dst=[296, 0, 302, 2152, 138, 242, 6050, 6] combine_rows_per_src=[6604, 4972, 3950, 8182, 8192, 8190, 8068, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=342931456 dispatch_meta_bytes=215488 combine_payload_bytes=342931456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=343146944 ingress_bytes=453031936 link_bytes=453031936 token_bytes=686078400 weight_movement_bytes=110100480 total_nvlink_bytes=796178880 weight_to_token_ratio=0.160478 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.878970477e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55514 payload_rows_remote=48526 dup_rows=10022 dup_rows_remote=6752 dedup_ratio=0.152924 router_tokens_per_expert=[0, 0, 0, 332, 3802, 0, 0, 0, 0, 0, 0, 0, 0, 2201, 0, 0, 0, 1212, 0, 0, 0, 0, 4, 774, 4092, 0, 0, 0, 0, 0, 1, 1510, 0, 160, 0, 0, 0, 4096, 3246, 0, 13, 0, 0, 0, 4096, 162, 0, 0, 678, 0, 0, 2280, 0, 0, 0, 13, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7648, 4402, 3710, 8190, 8192, 8192, 6988, 8192] dup_rows_per_dst=[620, 0, 270, 3016, 2496, 350, 3270, 0] combine_rows_per_src=[7648, 4402, 3710, 8190, 8192, 8192, 6988, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=330201088 dispatch_meta_bytes=228320 combine_payload_bytes=330201088 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=330429408 ingress_bytes=660502528 link_bytes=660502528 token_bytes=660630496 weight_movement_bytes=330301440 total_nvlink_bytes=990931936 weight_to_token_ratio=0.499979 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.148722530e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57254 payload_rows_remote=49684 dup_rows=8282 dup_rows_remote=5656 dedup_ratio=0.126373 router_tokens_per_expert=[0, 2, 0, 679, 3642, 0, 0, 0, 49, 0, 0, 0, 0, 3031, 0, 0, 0, 565, 0, 0, 0, 0, 4, 375, 4087, 0, 9, 0, 0, 0, 23, 2131, 0, 150, 0, 0, 0, 4096, 2504, 0, 9, 0, 1, 0, 4051, 182, 0, 0, 888, 0, 0, 2055, 0, 0, 0, 133, 8, 4089, 5, 0, 0, 0, 0, 0] payload_rows_per_dst=[7534, 7052, 8192, 7570, 5052, 8126, 5546, 8182] dup_rows_per_dst=[1112, 1412, 1888, 2626, 256, 360, 606, 22] combine_rows_per_src=[7534, 7052, 8192, 7570, 5052, 8126, 5546, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=345196544 dispatch_meta_bytes=229352 combine_payload_bytes=345196544 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=345425896 ingress_bytes=455297024 link_bytes=455297024 token_bytes=690622440 weight_movement_bytes=110100480 total_nvlink_bytes=800722920 weight_to_token_ratio=0.159422 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.918364082e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=347834368 dispatch_meta_bytes=221112 combine_payload_bytes=347834368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=348055480 ingress_bytes=457934848 link_bytes=457934848 token_bytes=695889848 weight_movement_bytes=110100480 total_nvlink_bytes=805990328 weight_to_token_ratio=0.158215 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.964240180e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52616 payload_rows_remote=44942 dup_rows=12920 dup_rows_remote=10788 dedup_ratio=0.197144 router_tokens_per_expert=[0, 0, 1, 89, 3668, 0, 1, 0, 0, 1, 0, 0, 0, 2085, 0, 0, 0, 1502, 0, 0, 0, 0, 3, 467, 4088, 1, 0, 0, 0, 0, 0, 889, 0, 140, 0, 0, 0, 4096, 3216, 0, 0, 0, 0, 0, 4092, 167, 0, 0, 1400, 0, 0, 2759, 0, 0, 1, 7, 0, 4095, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7674, 8192, 8192, 6914, 682, 5984, 6808, 8170] dup_rows_per_dst=[2132, 4156, 3944, 932, 18, 212, 1526, 0] combine_rows_per_src=[7674, 8192, 8192, 6914, 682, 5984, 6808, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=356134912 dispatch_meta_bytes=221360 combine_payload_bytes=356134912 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=356356272 ingress_bytes=576335872 link_bytes=576335872 token_bytes=712491184 weight_movement_bytes=220200960 total_nvlink_bytes=932692144 weight_to_token_ratio=0.309058 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.002342872e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54552 payload_rows_remote=46360 dup_rows=10984 dup_rows_remote=6390 dedup_ratio=0.167603 router_tokens_per_expert=[0, 0, 0, 224, 3605, 0, 0, 0, 2, 0, 0, 0, 0, 2519, 0, 0, 0, 1402, 0, 0, 0, 0, 0, 895, 4079, 0, 0, 1, 0, 0, 14, 1387, 0, 179, 0, 0, 0, 4096, 2724, 0, 0, 0, 0, 0, 4094, 101, 0, 0, 619, 0, 0, 2729, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7278, 8192, 8192, 2794, 5528, 8188, 6188, 8192] dup_rows_per_dst=[380, 5008, 4594, 10, 278, 202, 512, 0] combine_rows_per_src=[7278, 8192, 8192, 2794, 5528, 8188, 6188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65276 payload_rows_remote=57170 dup_rows=260 dup_rows_remote=256 dedup_ratio=0.003967 router_tokens_per_expert=[0, 5, 4093, 9, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1212, 0, 0, 4092, 4046, 2979, 2, 0, 0, 0, 4093, 0, 4067, 0, 0, 0, 4052, 1, 0, 0, 2, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 0, 2, 0, 0, 3, 4] payload_rows_per_dst=[8186, 8192, 8178, 8138, 8106, 8190, 8094, 8192] dup_rows_per_dst=[28, 2, 204, 0, 4, 6, 2, 14] combine_rows_per_src=[8186, 8192, 8178, 8138, 8106, 8190, 8094, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65170 payload_rows_remote=56980 dup_rows=366 dup_rows_remote=362 dedup_ratio=0.005585 router_tokens_per_expert=[0, 0, 4092, 10, 0, 0, 4089, 0, 5, 0, 1, 0, 1, 1, 0, 0, 0, 0, 526, 1, 0, 4092, 4048, 3671, 0, 0, 0, 0, 4094, 0, 4058, 0, 0, 3, 4018, 0, 0, 0, 12, 0, 6, 2, 0, 17, 1, 0, 0, 1, 0, 3, 0, 11, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 1] payload_rows_per_dst=[8184, 8190, 8140, 8116, 8042, 8192, 8118, 8188] dup_rows_per_dst=[20, 4, 256, 0, 24, 50, 6, 6] combine_rows_per_src=[8184, 8190, 8140, 8116, 8042, 8192, 8118, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=322144256 dispatch_meta_bytes=222920 combine_payload_bytes=322144256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=322367176 ingress_bytes=432244736 link_bytes=432244736 token_bytes=644511432 weight_movement_bytes=110100480 total_nvlink_bytes=754611912 weight_to_token_ratio=0.170828 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.517446879e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65014 payload_rows_remote=56826 dup_rows=522 dup_rows_remote=456 dedup_ratio=0.007965 router_tokens_per_expert=[0, 2, 4083, 112, 0, 0, 4084, 0, 0, 0, 0, 6, 1, 0, 0, 5, 0, 0, 975, 0, 0, 4094, 4056, 3057, 0, 2, 1, 0, 4087, 0, 4082, 0, 0, 0, 4052, 3, 0, 0, 3, 0, 29, 0, 1, 10, 0, 0, 0, 0, 5, 0, 1, 13, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8172, 8182, 7894, 8166, 8110, 8188, 8114, 8188] dup_rows_per_dst=[222, 10, 170, 4, 6, 66, 36, 8] combine_rows_per_src=[8172, 8182, 7894, 8166, 8110, 8188, 8114, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=332308480 dispatch_meta_bytes=211000 combine_payload_bytes=332308480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=332519480 ingress_bytes=442408960 link_bytes=442408960 token_bytes=664827960 weight_movement_bytes=110100480 total_nvlink_bytes=774928440 weight_to_token_ratio=0.165607 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.694219452e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409794560 dispatch_meta_bytes=229704 combine_payload_bytes=409794560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410024264 ingress_bytes=519895040 link_bytes=519895040 token_bytes=819818824 weight_movement_bytes=110100480 total_nvlink_bytes=929919304 weight_to_token_ratio=0.134299 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.041829826e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63788 payload_rows_remote=55604 dup_rows=1748 dup_rows_remote=1748 dedup_ratio=0.026672 router_tokens_per_expert=[0, 0, 4095, 53, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1478, 1, 0, 4092, 4006, 2686, 1, 0, 0, 0, 4094, 0, 4044, 0, 0, 0, 4095, 2, 1, 1, 0, 0, 15, 0, 0, 4, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7294, 8192, 7618, 8090, 8192, 8192, 8026, 8184] dup_rows_per_dst=[74, 2, 368, 212, 408, 38, 646, 0] combine_rows_per_src=[7294, 8192, 7618, 8090, 8192, 8192, 8026, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=408432640 dispatch_meta_bytes=229368 combine_payload_bytes=408432640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=408662008 ingress_bytes=518533120 link_bytes=518533120 token_bytes=817094648 weight_movement_bytes=110100480 total_nvlink_bytes=927195128 weight_to_token_ratio=0.134746 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.018143797e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=407328768 dispatch_meta_bytes=229128 combine_payload_bytes=407328768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407557896 ingress_bytes=517429248 link_bytes=517429248 token_bytes=814886664 weight_movement_bytes=110100480 total_nvlink_bytes=924987144 weight_to_token_ratio=0.135111 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.998945648e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65308 payload_rows_remote=57130 dup_rows=228 dup_rows_remote=228 dedup_ratio=0.003479 router_tokens_per_expert=[0, 0, 4096, 70, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 779, 0, 0, 4096, 4089, 3282, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8044, 8188, 8130, 8192, 8178, 8192] dup_rows_per_dst=[140, 0, 78, 0, 0, 10, 0, 0] combine_rows_per_src=[8192, 8192, 8044, 8188, 8130, 8192, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65108 payload_rows_remote=56966 dup_rows=428 dup_rows_remote=428 dedup_ratio=0.006531 router_tokens_per_expert=[0, 0, 4094, 111, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1643, 0, 0, 4096, 4037, 2412, 0, 0, 0, 0, 4096, 0, 4071, 0, 0, 0, 4085, 0, 0, 0, 1, 0, 15, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8188, 8192, 7958, 8142, 8170, 8192, 8074, 8192] dup_rows_per_dst=[222, 0, 152, 0, 2, 50, 0, 2] combine_rows_per_src=[8188, 8192, 7958, 8142, 8170, 8192, 8074, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=398569472 dispatch_meta_bytes=229408 combine_payload_bytes=398569472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398798880 ingress_bytes=508669952 link_bytes=508669952 token_bytes=797368352 weight_movement_bytes=110100480 total_nvlink_bytes=907468832 weight_to_token_ratio=0.138080 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.846607084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409507840 dispatch_meta_bytes=229432 combine_payload_bytes=409507840 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=409737272 ingress_bytes=629708800 link_bytes=629708800 token_bytes=819245112 weight_movement_bytes=220200960 total_nvlink_bytes=1039446072 weight_to_token_ratio=0.268785 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.095167173e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=408332288 dispatch_meta_bytes=229576 combine_payload_bytes=408332288 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=408561864 ingress_bytes=628533248 link_bytes=628533248 token_bytes=816894152 weight_movement_bytes=220200960 total_nvlink_bytes=1037095112 weight_to_token_ratio=0.269559 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.093122694e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64996 payload_rows_remote=56822 dup_rows=540 dup_rows_remote=434 dedup_ratio=0.008240 router_tokens_per_expert=[0, 0, 4086, 51, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1001, 0, 0, 4095, 4051, 3152, 0, 0, 0, 0, 4095, 0, 4034, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 8, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8174, 8186, 8098, 8110, 8188, 7958, 8104, 8178] dup_rows_per_dst=[106, 0, 222, 208, 0, 4, 0, 0] combine_rows_per_src=[8174, 8186, 8098, 8110, 8188, 7958, 8104, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65264 payload_rows_remote=57220 dup_rows=272 dup_rows_remote=160 dedup_ratio=0.004150 router_tokens_per_expert=[0, 0, 4095, 72, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 803, 0, 0, 4096, 4069, 3275, 0, 0, 0, 0, 4096, 0, 4081, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8044, 8162, 8154, 8192, 8138, 8192] dup_rows_per_dst=[144, 0, 112, 0, 0, 16, 0, 0] combine_rows_per_src=[8190, 8192, 8044, 8162, 8154, 8192, 8138, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60756 payload_rows_remote=52564 dup_rows=4780 dup_rows_remote=4500 dedup_ratio=0.072937 router_tokens_per_expert=[0, 2, 7, 1, 0, 0, 0, 0, 0, 0, 0, 1065, 0, 219, 0, 0, 0, 0, 3467, 403, 0, 0, 0, 2, 784, 0, 0, 0, 0, 4084, 4030, 0, 0, 1, 4096, 0, 0, 139, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 1, 3058, 2, 0, 0, 0, 0, 0, 4087, 2, 0, 3216, 1, 1, 3] payload_rows_per_dst=[8176, 8186, 7370, 8076, 8192, 8192, 6122, 6442] dup_rows_per_dst=[12, 2556, 374, 1552, 280, 0, 2, 4] combine_rows_per_src=[8176, 8186, 7370, 8076, 8192, 8192, 6122, 6442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60598 payload_rows_remote=52406 dup_rows=4938 dup_rows_remote=3782 dedup_ratio=0.075348 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4, 1, 0, 4, 0, 319, 0, 259, 0, 0, 0, 0, 3928, 1039, 0, 0, 0, 0, 670, 0, 0, 0, 0, 4094, 4056, 0, 0, 0, 4093, 0, 0, 308, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 3219, 0, 0, 0, 1, 0, 1, 4092, 5, 3, 2562, 0, 1, 10] payload_rows_per_dst=[8192, 8192, 8116, 8130, 8190, 8192, 6440, 5146] dup_rows_per_dst=[6, 1156, 1818, 1322, 618, 0, 0, 18] combine_rows_per_src=[8192, 8192, 8116, 8130, 8190, 8192, 6440, 5146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63076 payload_rows_remote=54884 dup_rows=2460 dup_rows_remote=2460 dedup_ratio=0.037537 router_tokens_per_expert=[0, 0, 0, 0, 0, 8, 0, 0, 0, 1, 0, 182, 0, 370, 0, 1, 0, 0, 3804, 444, 0, 2, 0, 0, 353, 0, 0, 0, 0, 4089, 4080, 0, 1, 0, 4092, 0, 1, 77, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 3540, 3, 2, 2, 0, 0, 0, 4090, 13, 0, 3512, 0, 1, 1] payload_rows_per_dst=[8188, 8190, 8006, 8166, 8190, 8192, 7092, 7052] dup_rows_per_dst=[6, 1098, 494, 700, 154, 0, 6, 2] combine_rows_per_src=[8188, 8190, 8006, 8166, 8190, 8192, 7092, 7052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48668 payload_rows_remote=40476 dup_rows=16868 dup_rows_remote=8196 dedup_ratio=0.257385 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 567, 0, 185, 0, 0, 0, 0, 3763, 14, 0, 0, 0, 0, 753, 0, 0, 0, 0, 4092, 4034, 0, 0, 0, 4095, 0, 0, 239, 0, 1, 0, 4095, 0, 0, 1, 2, 0, 1, 0, 1, 3487, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3339, 0, 0, 3] payload_rows_per_dst=[8188, 1492, 4718, 1506, 8192, 8192, 8188, 8192] dup_rows_per_dst=[0, 12, 16, 0, 1306, 6, 6856, 8672] combine_rows_per_src=[8188, 1492, 4718, 1506, 8192, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=407300096 dispatch_meta_bytes=229024 combine_payload_bytes=407300096 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=407529120 ingress_bytes=847702016 link_bytes=847702016 token_bytes=814829216 weight_movement_bytes=440401920 total_nvlink_bytes=1255231136 weight_to_token_ratio=0.540484 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.474293229e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=410152960 dispatch_meta_bytes=229520 combine_payload_bytes=410152960 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=410382480 ingress_bytes=850554880 link_bytes=850554880 token_bytes=820535440 weight_movement_bytes=440401920 total_nvlink_bytes=1260937360 weight_to_token_ratio=0.536725 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.479254829e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=376778752 dispatch_meta_bytes=228256 combine_payload_bytes=376778752 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=377007008 ingress_bytes=596979712 link_bytes=596979712 token_bytes=753785760 weight_movement_bytes=220200960 total_nvlink_bytes=973986720 weight_to_token_ratio=0.292127 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.038245905e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=375646208 dispatch_meta_bytes=224752 combine_payload_bytes=375646208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375870960 ingress_bytes=485746688 link_bytes=485746688 token_bytes=751517168 weight_movement_bytes=110100480 total_nvlink_bytes=861617648 weight_to_token_ratio=0.146504 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=26368 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.447933820e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=393408512 dispatch_meta_bytes=229376 combine_payload_bytes=393408512 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=393637888 ingress_bytes=503508992 link_bytes=503508992 token_bytes=787046400 weight_movement_bytes=110100480 total_nvlink_bytes=897146880 weight_to_token_ratio=0.139891 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.756849501e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55428 payload_rows_remote=48288 dup_rows=10108 dup_rows_remote=6144 dedup_ratio=0.154236 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 729, 0, 120, 0, 0, 0, 0, 3678, 415, 0, 0, 0, 0, 1473, 0, 0, 0, 0, 4077, 4037, 0, 0, 0, 4096, 0, 0, 280, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2337, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3334, 0, 0, 0] payload_rows_per_dst=[8154, 4436, 7576, 4028, 8192, 8192, 7140, 7710] dup_rows_per_dst=[0, 746, 610, 562, 560, 0, 3964, 3666] combine_rows_per_src=[8154, 4436, 7576, 4028, 8192, 8192, 7140, 7710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=290131968 dispatch_meta_bytes=194688 combine_payload_bytes=290131968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=290326656 ingress_bytes=510332928 link_bytes=510332928 token_bytes=580458624 weight_movement_bytes=220200960 total_nvlink_bytes=800659584 weight_to_token_ratio=0.379357 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=25344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.875528971e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61812 payload_rows_remote=53828 dup_rows=3724 dup_rows_remote=3138 dedup_ratio=0.056824 router_tokens_per_expert=[0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 680, 0, 188, 0, 0, 0, 0, 3974, 686, 0, 0, 0, 0, 304, 0, 0, 0, 0, 4091, 3981, 0, 0, 0, 4096, 0, 0, 99, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 3494, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 2977, 0, 0, 0] payload_rows_per_dst=[8182, 8192, 8124, 7984, 8192, 8192, 6990, 5956] dup_rows_per_dst=[10, 1734, 1196, 586, 198, 0, 0, 0] combine_rows_per_src=[8182, 8192, 8124, 7984, 8192, 8192, 6990, 5956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=346128384 dispatch_meta_bytes=217728 combine_payload_bytes=346128384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=346346112 ingress_bytes=456228864 link_bytes=456228864 token_bytes=692474496 weight_movement_bytes=110100480 total_nvlink_bytes=802574976 weight_to_token_ratio=0.158996 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=25600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.934570313e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=385839104 dispatch_meta_bytes=227864 combine_payload_bytes=385839104 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=386066968 ingress_bytes=826241024 link_bytes=826241024 token_bytes=771906072 weight_movement_bytes=440401920 total_nvlink_bytes=1212307992 weight_to_token_ratio=0.570538 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=25088 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.436969034e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63738 payload_rows_remote=55548 dup_rows=1798 dup_rows_remote=1798 dedup_ratio=0.027435 router_tokens_per_expert=[4093, 0, 625, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 2, 3, 0, 0, 0, 0, 3386, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 280, 0, 0, 0, 0, 0, 0, 0, 122, 1, 0, 1, 4095, 4096, 0, 9, 0, 0, 0, 9, 0, 1, 4025, 0] payload_rows_per_dst=[7952, 8188, 8192, 6778, 8190, 8190, 8192, 8056] dup_rows_per_dst=[952, 0, 6, 0, 0, 560, 266, 14] combine_rows_per_src=[7952, 8188, 8192, 6778, 8190, 8190, 8192, 8056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57292 payload_rows_remote=49124 dup_rows=8244 dup_rows_remote=8178 dedup_ratio=0.125793 router_tokens_per_expert=[1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 597, 0, 359, 0, 0, 0, 0, 3900, 457, 0, 0, 0, 0, 609, 0, 1, 0, 0, 4081, 4055, 0, 0, 0, 4095, 0, 0, 284, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 3001, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 3135, 0, 0, 0] payload_rows_per_dst=[8168, 8190, 8022, 8140, 4314, 8186, 6002, 6270] dup_rows_per_dst=[66, 1912, 698, 5348, 218, 0, 0, 2] combine_rows_per_src=[8168, 8190, 8022, 8140, 4314, 8186, 6002, 6270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61990 payload_rows_remote=54054 dup_rows=3546 dup_rows_remote=3084 dedup_ratio=0.054108 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 420, 0, 0, 0, 0, 3845, 354, 0, 0, 0, 0, 646, 0, 0, 0, 0, 4094, 4072, 0, 0, 0, 4096, 0, 0, 274, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3367, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3202, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 7936, 8152, 8192, 8192, 6734, 6404] dup_rows_per_dst=[0, 1252, 462, 1284, 548, 0, 0, 0] combine_rows_per_src=[8188, 8192, 7936, 8152, 8192, 8192, 6734, 6404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63930 payload_rows_remote=55746 dup_rows=1606 dup_rows_remote=1606 dedup_ratio=0.024506 router_tokens_per_expert=[4096, 0, 702, 0, 3656, 2, 0, 3, 0, 0, 0, 1, 4090, 1, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 1, 1, 0, 0, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 1, 0, 3, 0, 0, 0, 0, 206, 0, 0, 0, 4094, 4088, 0, 46, 0, 0, 0, 8, 0, 0, 3995, 0] payload_rows_per_dst=[7884, 8184, 8162, 7120, 8192, 8192, 8192, 8004] dup_rows_per_dst=[842, 0, 0, 4, 0, 270, 488, 2] combine_rows_per_src=[7884, 8184, 8162, 7120, 8192, 8192, 8192, 8004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61540 payload_rows_remote=53348 dup_rows=3996 dup_rows_remote=3832 dedup_ratio=0.060974 router_tokens_per_expert=[4095, 0, 1131, 1, 4017, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 3, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 1, 0, 0, 1, 0, 688, 0, 29, 0, 4094, 4087, 4, 103, 0, 1, 0, 11, 0, 0, 4062, 0] payload_rows_per_dst=[8120, 8192, 8150, 4372, 8190, 8192, 8192, 8132] dup_rows_per_dst=[2178, 8, 0, 0, 0, 164, 1630, 16] combine_rows_per_src=[8120, 8192, 8150, 4372, 8190, 8192, 8192, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57036 payload_rows_remote=48864 dup_rows=8500 dup_rows_remote=7680 dedup_ratio=0.129700 router_tokens_per_expert=[4095, 0, 1016, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2954, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 108, 0, 0, 2, 1, 0, 0, 1, 151, 0, 0, 0, 4094, 4095, 0, 33, 0, 0, 0, 0, 0, 0, 4082, 1] payload_rows_per_dst=[2032, 8190, 8190, 8182, 8192, 5888, 8190, 8172] dup_rows_per_dst=[0, 0, 2, 5608, 4, 146, 1920, 820] combine_rows_per_src=[2032, 8190, 8190, 8182, 8192, 5888, 8190, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=398168064 dispatch_meta_bytes=229384 combine_payload_bytes=398168064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398397448 ingress_bytes=508268544 link_bytes=508268544 token_bytes=796565512 weight_movement_bytes=110100480 total_nvlink_bytes=906665992 weight_to_token_ratio=0.138219 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.839625938e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=352120832 dispatch_meta_bytes=229208 combine_payload_bytes=352120832 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=352350040 ingress_bytes=572321792 link_bytes=572321792 token_bytes=704470872 weight_movement_bytes=220200960 total_nvlink_bytes=924671832 weight_to_token_ratio=0.312576 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.953617270e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=387459072 dispatch_meta_bytes=228552 combine_payload_bytes=387459072 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=387687624 ingress_bytes=717760512 link_bytes=717760512 token_bytes=775146696 weight_movement_bytes=330301440 total_nvlink_bytes=1105448136 weight_to_token_ratio=0.426115 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.248303582e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=399587328 dispatch_meta_bytes=229408 combine_payload_bytes=399587328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399816736 ingress_bytes=509687808 link_bytes=509687808 token_bytes=799404064 weight_movement_bytes=110100480 total_nvlink_bytes=909504544 weight_to_token_ratio=0.137728 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.864309274e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=382398464 dispatch_meta_bytes=228720 combine_payload_bytes=382398464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382627184 ingress_bytes=492498944 link_bytes=492498944 token_bytes=765025648 weight_movement_bytes=110100480 total_nvlink_bytes=875126128 weight_to_token_ratio=0.143917 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.565366658e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59818 payload_rows_remote=51626 dup_rows=5718 dup_rows_remote=5324 dedup_ratio=0.087250 router_tokens_per_expert=[4096, 0, 431, 0, 3784, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 14, 0, 0, 0, 0, 171, 0, 0, 0, 4096, 4096, 0, 26, 0, 0, 0, 0, 0, 0, 4089, 0] payload_rows_per_dst=[2708, 8192, 8192, 7972, 8192, 8192, 8192, 8178] dup_rows_per_dst=[66, 0, 0, 4728, 0, 530, 394, 0] combine_rows_per_src=[2708, 8192, 8192, 7972, 8192, 8192, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=350257152 dispatch_meta_bytes=226176 combine_payload_bytes=350257152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=350483328 ingress_bytes=460357632 link_bytes=460357632 token_bytes=700740480 weight_movement_bytes=110100480 total_nvlink_bytes=810840960 weight_to_token_ratio=0.157120 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.006376379e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63214 payload_rows_remote=56680 dup_rows=2322 dup_rows_remote=2322 dedup_ratio=0.035431 router_tokens_per_expert=[4096, 0, 839, 0, 3755, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 283, 0, 0, 0, 4085, 4096, 0, 168, 0, 0, 0, 0, 0, 0, 3899, 0] payload_rows_per_dst=[7942, 8192, 8192, 6534, 8192, 8172, 8192, 7798] dup_rows_per_dst=[1246, 0, 0, 0, 0, 174, 902, 0] combine_rows_per_src=[7942, 8192, 8192, 6534, 8192, 8172, 8192, 7798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=370055168 dispatch_meta_bytes=227800 combine_payload_bytes=370055168 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=370282968 ingress_bytes=920557568 link_bytes=920557568 token_bytes=740338136 weight_movement_bytes=550502400 total_nvlink_bytes=1290840536 weight_to_token_ratio=0.743582 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=29184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.601001016e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63468 payload_rows_remote=55278 dup_rows=2068 dup_rows_remote=2066 dedup_ratio=0.031555 router_tokens_per_expert=[1, 4095, 4090, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 3266, 2, 1, 0, 0, 4094, 0, 0, 0, 58, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 346, 0, 3573, 218, 0, 278, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4094, 0, 1, 0, 455] payload_rows_per_dst=[8182, 6532, 8188, 8190, 8190, 7806, 8190, 8190] dup_rows_per_dst=[2, 6, 116, 4, 2, 468, 556, 914] combine_rows_per_src=[8182, 6532, 8188, 8190, 8190, 7806, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=406282240 dispatch_meta_bytes=236008 combine_payload_bytes=406282240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406518248 ingress_bytes=516382720 link_bytes=516382720 token_bytes=812800488 weight_movement_bytes=110100480 total_nvlink_bytes=922900968 weight_to_token_ratio=0.135458 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=26112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.980744804e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58378 payload_rows_remote=50192 dup_rows=7158 dup_rows_remote=350 dedup_ratio=0.109222 router_tokens_per_expert=[4096, 0, 443, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 1, 0, 0, 0, 0, 101, 1, 0, 0, 4096, 4095, 0, 12, 0, 1, 0, 0, 0, 0, 4077, 0] payload_rows_per_dst=[8186, 8190, 8190, 7216, 8192, 8192, 2056, 8156] dup_rows_per_dst=[6808, 8, 70, 0, 0, 234, 38, 0] combine_rows_per_src=[8186, 8190, 8190, 7216, 8192, 8192, 2056, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64466 payload_rows_remote=57114 dup_rows=1070 dup_rows_remote=1070 dedup_ratio=0.016327 router_tokens_per_expert=[0, 4096, 4084, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 3669, 6, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 223, 0, 3753, 126, 0, 131, 0, 0, 1, 0, 1, 2, 0, 0, 0, 2, 4092, 0, 0, 0, 290] payload_rows_per_dst=[8170, 7352, 8188, 8192, 8192, 7990, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 0, 10, 214, 268, 578] combine_rows_per_src=[8170, 7352, 8188, 8192, 8192, 7990, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64150 payload_rows_remote=55960 dup_rows=1386 dup_rows_remote=1386 dedup_ratio=0.021149 router_tokens_per_expert=[4096, 0, 503, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 197, 0, 8, 0, 4096, 4096, 0, 45, 0, 0, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[7982, 8192, 8190, 7022, 8192, 8192, 8192, 8188] dup_rows_per_dst=[638, 0, 0, 0, 0, 248, 500, 0] combine_rows_per_src=[7982, 8192, 8190, 7022, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63320 payload_rows_remote=55400 dup_rows=2216 dup_rows_remote=1572 dedup_ratio=0.033813 router_tokens_per_expert=[0, 4096, 4094, 0, 0, 0, 0, 2, 4092, 0, 8, 0, 0, 3116, 3, 2, 0, 0, 4096, 0, 0, 2, 10, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 398, 0, 3682, 202, 0, 670, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 1, 97] payload_rows_per_dst=[8192, 6254, 8192, 8192, 8192, 7920, 8186, 8192] dup_rows_per_dst=[0, 4, 24, 8, 0, 644, 1340, 196] combine_rows_per_src=[8192, 6254, 8192, 8192, 8192, 7920, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61664 payload_rows_remote=53474 dup_rows=3872 dup_rows_remote=810 dedup_ratio=0.059082 router_tokens_per_expert=[0, 4095, 4095, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 3133, 0, 0, 0, 1, 4096, 0, 1, 0, 4, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 5, 0, 0, 0, 0, 265, 1, 3619, 190, 0, 230, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 744] payload_rows_per_dst=[8190, 6270, 7898, 8190, 8192, 6544, 8190, 8190] dup_rows_per_dst=[0, 0, 14, 2, 20, 314, 460, 3062] combine_rows_per_src=[8190, 6270, 7898, 8190, 8192, 6544, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=396232704 dispatch_meta_bytes=229376 combine_payload_bytes=396232704 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=396462080 ingress_bytes=616433664 link_bytes=616433664 token_bytes=792694784 weight_movement_bytes=220200960 total_nvlink_bytes=1012895744 weight_to_token_ratio=0.277788 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.072079528e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=359776256 dispatch_meta_bytes=202168 combine_payload_bytes=359776256 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=359978424 ingress_bytes=690077696 link_bytes=690077696 token_bytes=719754680 weight_movement_bytes=330301440 total_nvlink_bytes=1050056120 weight_to_token_ratio=0.458908 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=24064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.200158612e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=409393152 dispatch_meta_bytes=232736 combine_payload_bytes=409393152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=409625888 ingress_bytes=629594112 link_bytes=629594112 token_bytes=819019040 weight_movement_bytes=220200960 total_nvlink_bytes=1039220000 weight_to_token_ratio=0.268859 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=27264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.094967711e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=401121280 dispatch_meta_bytes=229384 combine_payload_bytes=401121280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401350664 ingress_bytes=511221760 link_bytes=511221760 token_bytes=802471944 weight_movement_bytes=110100480 total_nvlink_bytes=912572424 weight_to_token_ratio=0.137202 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.890987222e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63724 payload_rows_remote=55532 dup_rows=1812 dup_rows_remote=1398 dedup_ratio=0.027649 router_tokens_per_expert=[0, 4096, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3586, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 3268, 173, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 550] payload_rows_per_dst=[8188, 7172, 8192, 8192, 8192, 7404, 8192, 8192] dup_rows_per_dst=[0, 0, 26, 0, 0, 272, 414, 1100] combine_rows_per_src=[8188, 7172, 8192, 8192, 8192, 7404, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=397107200 dispatch_meta_bytes=227888 combine_payload_bytes=397107200 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=397335088 ingress_bytes=617308160 link_bytes=617308160 token_bytes=794442288 weight_movement_bytes=220200960 total_nvlink_bytes=1014643248 weight_to_token_ratio=0.277177 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=28544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.073600420e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=383301632 dispatch_meta_bytes=217136 combine_payload_bytes=383301632 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=383518768 ingress_bytes=603502592 link_bytes=603502592 token_bytes=766820400 weight_movement_bytes=220200960 total_nvlink_bytes=987021360 weight_to_token_ratio=0.287161 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.049590267e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64682 payload_rows_remote=56490 dup_rows=854 dup_rows_remote=854 dedup_ratio=0.013031 router_tokens_per_expert=[0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 3730, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 3758, 205, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 56] payload_rows_per_dst=[8192, 7478, 8192, 8192, 8192, 8052, 8192, 8192] dup_rows_per_dst=[0, 0, 20, 0, 0, 262, 460, 112] combine_rows_per_src=[8192, 7478, 8192, 8192, 8192, 8052, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=398053376 dispatch_meta_bytes=227720 combine_payload_bytes=398053376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398281096 ingress_bytes=508153856 link_bytes=508153856 token_bytes=796334472 weight_movement_bytes=110100480 total_nvlink_bytes=906434952 weight_to_token_ratio=0.138259 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.837631325e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62788 payload_rows_remote=54596 dup_rows=2748 dup_rows_remote=2744 dedup_ratio=0.041931 router_tokens_per_expert=[0, 4093, 0, 0, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 756, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 2, 0, 2, 0, 12, 0, 0, 0, 11, 323, 2846, 0, 1, 0, 4088, 4094, 4096, 0, 1, 0, 3786, 0] payload_rows_per_dst=[8186, 8190, 8190, 8050, 8192, 8192, 6214, 7574] dup_rows_per_dst=[0, 1100, 0, 1476, 4, 20, 148, 0] combine_rows_per_src=[8186, 8190, 8190, 8050, 8192, 8192, 6214, 7574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61748 payload_rows_remote=53560 dup_rows=3788 dup_rows_remote=3132 dedup_ratio=0.057800 router_tokens_per_expert=[0, 4095, 0, 0, 0, 0, 6, 0, 326, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 1123, 1, 0, 0, 0, 3860, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 2, 0, 8, 0, 2, 0, 0, 0, 0, 1675, 2080, 1, 0, 0, 4089, 4085, 4096, 0, 0, 0, 3208, 0] payload_rows_per_dst=[8192, 8188, 8186, 7784, 8192, 8192, 6598, 6416] dup_rows_per_dst=[10, 656, 0, 2184, 14, 10, 914, 0] combine_rows_per_src=[8192, 8188, 8186, 7784, 8192, 8192, 6598, 6416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54402 payload_rows_remote=46210 dup_rows=11134 dup_rows_remote=11098 dedup_ratio=0.169891 router_tokens_per_expert=[0, 4095, 4094, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 3339, 1, 0, 0, 0, 4096, 1, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 469, 0, 3905, 144, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 288] payload_rows_per_dst=[8192, 7862, 8192, 8182, 8162, 8130, 4110, 1572] dup_rows_per_dst=[36, 9984, 18, 0, 0, 906, 64, 126] combine_rows_per_src=[8192, 7862, 8192, 8182, 8162, 8130, 4110, 1572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63752 payload_rows_remote=55560 dup_rows=1784 dup_rows_remote=1700 dedup_ratio=0.027222 router_tokens_per_expert=[0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3311, 0, 0, 0, 0, 4096, 0, 0, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 0, 3739, 205, 0, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 269] payload_rows_per_dst=[8192, 6622, 8192, 8192, 8192, 7978, 8192, 8192] dup_rows_per_dst=[0, 0, 84, 0, 0, 632, 530, 538] combine_rows_per_src=[8192, 6622, 8192, 8192, 8192, 7978, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=404920320 dispatch_meta_bytes=229376 combine_payload_bytes=404920320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405149696 ingress_bytes=515020800 link_bytes=515020800 token_bytes=810070016 weight_movement_bytes=110100480 total_nvlink_bytes=920170496 weight_to_token_ratio=0.135915 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.957058776e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60836 payload_rows_remote=52646 dup_rows=4700 dup_rows_remote=4632 dedup_ratio=0.071716 router_tokens_per_expert=[1, 4095, 3, 0, 0, 0, 0, 0, 1090, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 1163, 0, 0, 0, 0, 3852, 1, 0, 0, 2, 0, 1, 1, 0, 0, 0, 5, 9, 0, 9, 0, 12, 0, 0, 0, 3, 467, 2190, 0, 11, 0, 4094, 4083, 4092, 0, 1, 0, 3487, 0] payload_rows_per_dst=[8192, 8178, 8190, 7816, 8190, 8190, 5104, 6976] dup_rows_per_dst=[6, 2170, 0, 2216, 2, 68, 238, 0] combine_rows_per_src=[8192, 8178, 8190, 7816, 8190, 8190, 5104, 6976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=391344128 dispatch_meta_bytes=229360 combine_payload_bytes=391344128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=391573488 ingress_bytes=501444608 link_bytes=501444608 token_bytes=782917616 weight_movement_bytes=110100480 total_nvlink_bytes=893018096 weight_to_token_ratio=0.140628 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.720946468e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51230 payload_rows_remote=43038 dup_rows=14306 dup_rows_remote=3130 dedup_ratio=0.218292 router_tokens_per_expert=[0, 4096, 0, 0, 0, 1, 0, 0, 565, 0, 0, 0, 1, 1, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1165, 0, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 1, 0, 0, 1, 0, 0, 627, 2349, 0, 0, 0, 4094, 4095, 4095, 0, 0, 0, 3598, 0] payload_rows_per_dst=[8182, 1134, 8190, 2332, 8172, 8024, 7004, 8192] dup_rows_per_dst=[2, 4, 8, 0, 0, 8, 3108, 11176] combine_rows_per_src=[8182, 1134, 8190, 2332, 8172, 8024, 7004, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=383918080 dispatch_meta_bytes=226768 combine_payload_bytes=383918080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=384144848 ingress_bytes=494018560 link_bytes=494018560 token_bytes=768062928 weight_movement_bytes=110100480 total_nvlink_bytes=878163408 weight_to_token_ratio=0.143348 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.591795279e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=331233280 dispatch_meta_bytes=229232 combine_payload_bytes=331233280 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=331462512 ingress_bytes=551434240 link_bytes=551434240 token_bytes=662695792 weight_movement_bytes=220200960 total_nvlink_bytes=882896752 weight_to_token_ratio=0.332281 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.590348387e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58554 payload_rows_remote=50898 dup_rows=6982 dup_rows_remote=2916 dedup_ratio=0.106537 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1296, 0, 0, 0, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 2841, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 3391, 0] payload_rows_per_dst=[8192, 7110, 8192, 4036, 8192, 8192, 7656, 6984] dup_rows_per_dst=[0, 862, 0, 868, 0, 0, 4066, 1186] combine_rows_per_src=[8192, 7110, 8192, 4036, 8192, 8192, 7656, 6984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=398254080 dispatch_meta_bytes=229040 combine_payload_bytes=398254080 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=398483120 ingress_bytes=618455040 link_bytes=618455040 token_bytes=796737200 weight_movement_bytes=220200960 total_nvlink_bytes=1016938160 weight_to_token_ratio=0.276378 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.075595033e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=377366528 dispatch_meta_bytes=229112 combine_payload_bytes=377366528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=377595640 ingress_bytes=487467008 link_bytes=487467008 token_bytes=754962168 weight_movement_bytes=110100480 total_nvlink_bytes=865062648 weight_to_token_ratio=0.145836 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.477853015e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=308496384 dispatch_meta_bytes=184672 combine_payload_bytes=308496384 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=308681056 ingress_bytes=748898304 link_bytes=748898304 token_bytes=617177440 weight_movement_bytes=440401920 total_nvlink_bytes=1057579360 weight_to_token_ratio=0.713574 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=26112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.302457323e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62358 payload_rows_remote=55026 dup_rows=3178 dup_rows_remote=2122 dedup_ratio=0.048492 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 383, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 641, 0, 0, 0, 0, 3553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8, 0, 0, 0, 0, 0, 1, 1429, 2949, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 3318, 0] payload_rows_per_dst=[8192, 8192, 8192, 7332, 8192, 8192, 7430, 6636] dup_rows_per_dst=[0, 768, 0, 1056, 0, 26, 1328, 0] combine_rows_per_src=[8192, 8192, 8192, 7332, 8192, 8192, 7430, 6636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=364836864 dispatch_meta_bytes=215256 combine_payload_bytes=364836864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=365052120 ingress_bytes=474937344 link_bytes=474937344 token_bytes=729888984 weight_movement_bytes=110100480 total_nvlink_bytes=839989464 weight_to_token_ratio=0.150846 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=21120 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.259941550e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63412 payload_rows_remote=57074 dup_rows=2124 dup_rows_remote=2118 dedup_ratio=0.032410 router_tokens_per_expert=[0, 0, 12, 0, 4094, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 62, 0, 0, 0, 3090, 1, 19, 0, 0, 0, 0, 1, 0, 4041, 4096, 0, 8, 874, 0, 163, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4018] payload_rows_per_dst=[8192, 8192, 8192, 8184, 6338, 8084, 8192, 8038] dup_rows_per_dst=[22, 0, 4, 0, 6, 0, 2090, 2] combine_rows_per_src=[8192, 8192, 8192, 8184, 6338, 8084, 8192, 8038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60574 payload_rows_remote=52382 dup_rows=4962 dup_rows_remote=4956 dedup_ratio=0.075714 router_tokens_per_expert=[0, 0, 34, 0, 4087, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 5, 1, 1, 10, 2520, 0, 51, 4, 0, 0, 0, 0, 0, 3946, 4094, 0, 9, 1750, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846] payload_rows_per_dst=[8180, 8192, 8192, 8190, 8184, 8188, 3756, 7692] dup_rows_per_dst=[62, 6, 0, 0, 4884, 8, 2, 0] combine_rows_per_src=[8180, 8192, 8192, 8190, 8184, 8188, 3756, 7692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=394426368 dispatch_meta_bytes=228592 combine_payload_bytes=394426368 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=394654960 ingress_bytes=724727808 link_bytes=724727808 token_bytes=789081328 weight_movement_bytes=330301440 total_nvlink_bytes=1119382768 weight_to_token_ratio=0.418590 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=24448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.260420855e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62186 payload_rows_remote=53996 dup_rows=3350 dup_rows_remote=3350 dedup_ratio=0.051117 router_tokens_per_expert=[0, 4095, 0, 0, 0, 0, 0, 0, 603, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 4095, 0, 0, 1, 0, 464, 0, 0, 0, 0, 3907, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 1540, 2606, 0, 0, 0, 4095, 4095, 4095, 0, 0, 0, 3156, 0] payload_rows_per_dst=[8190, 8192, 8192, 7820, 8192, 8190, 7098, 6312] dup_rows_per_dst=[0, 1208, 4, 922, 2, 20, 1194, 0] combine_rows_per_src=[8190, 8192, 8192, 7820, 8192, 8190, 7098, 6312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=409106432 dispatch_meta_bytes=236768 combine_payload_bytes=409106432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409343200 ingress_bytes=519206912 link_bytes=519206912 token_bytes=818449632 weight_movement_bytes=110100480 total_nvlink_bytes=928550112 weight_to_token_ratio=0.134523 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=21632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.029862148e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59980 payload_rows_remote=51788 dup_rows=5556 dup_rows_remote=5556 dedup_ratio=0.084778 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1806, 0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 548, 3170, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 2042, 0] payload_rows_per_dst=[8192, 8192, 8192, 8058, 8192, 8192, 6878, 4084] dup_rows_per_dst=[0, 1392, 0, 3586, 0, 20, 558, 0] combine_rows_per_src=[8192, 8192, 8192, 8058, 8192, 8192, 6878, 4084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61782 payload_rows_remote=54296 dup_rows=3754 dup_rows_remote=3754 dedup_ratio=0.057281 router_tokens_per_expert=[3, 0, 33, 1, 4081, 0, 3, 4095, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 25, 0, 0, 8, 2715, 0, 42, 0, 0, 0, 0, 1, 0, 3742, 4089, 0, 7, 1751, 0, 85, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3893] payload_rows_per_dst=[8184, 8190, 8188, 8190, 5566, 7486, 8192, 7786] dup_rows_per_dst=[58, 4, 0, 2, 14, 0, 3676, 0] combine_rows_per_src=[8184, 8190, 8188, 8190, 5566, 7486, 8192, 7786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59296 payload_rows_remote=51440 dup_rows=6240 dup_rows_remote=3446 dedup_ratio=0.095215 router_tokens_per_expert=[0, 0, 6, 0, 4094, 0, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 44, 0, 0, 0, 2814, 1, 31, 9, 0, 0, 0, 0, 0, 3841, 4092, 0, 0, 1660, 0, 37, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 3847] payload_rows_per_dst=[7912, 8192, 8190, 8190, 5760, 5006, 8190, 7856] dup_rows_per_dst=[10, 2, 4, 12, 22, 8, 3388, 2794] combine_rows_per_src=[7912, 8192, 8190, 8190, 5760, 5006, 8190, 7856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=375474176 dispatch_meta_bytes=229352 combine_payload_bytes=375474176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375703528 ingress_bytes=485574656 link_bytes=485574656 token_bytes=751177704 weight_movement_bytes=110100480 total_nvlink_bytes=861278184 weight_to_token_ratio=0.146570 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.444941901e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7317501488 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=3659886128 ingress_bytes=5529323520 link_bytes=5529323520 total_nvlink_bytes=9189209648 weight_to_token_ratio=0.255785 prefetch_share_of_ingress=0.199121 dedup_ratio=0.104233 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037500 gemm_inner_pad_upper_bound=10668 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.616403019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=387043328 dispatch_meta_bytes=229384 combine_payload_bytes=387043328 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=387272712 ingress_bytes=607244288 link_bytes=607244288 token_bytes=774316040 weight_movement_bytes=220200960 total_nvlink_bytes=994517000 weight_to_token_ratio=0.284381 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056097691e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=371216384 dispatch_meta_bytes=229376 combine_payload_bytes=371216384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=371445760 ingress_bytes=481316864 link_bytes=481316864 token_bytes=742662144 weight_movement_bytes=110100480 total_nvlink_bytes=852762624 weight_to_token_ratio=0.148251 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.370891895e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61134 payload_rows_remote=52942 dup_rows=4402 dup_rows_remote=6 dedup_ratio=0.067169 router_tokens_per_expert=[0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 0, 2533, 0, 45, 0, 0, 0, 0, 0, 0, 3523, 4096, 0, 0, 2078, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5192, 7046, 8192, 7936] dup_rows_per_dst=[2, 0, 0, 0, 4, 0, 4396, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 5192, 7046, 8192, 7936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=389193728 dispatch_meta_bytes=232200 combine_payload_bytes=389193728 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=389425928 ingress_bytes=829595648 link_bytes=829595648 token_bytes=778619656 weight_movement_bytes=440401920 total_nvlink_bytes=1219021576 weight_to_token_ratio=0.565619 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.442803277e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=368721920 dispatch_meta_bytes=219544 combine_payload_bytes=368721920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368941464 ingress_bytes=478822400 link_bytes=478822400 token_bytes=737663384 weight_movement_bytes=110100480 total_nvlink_bytes=847763864 weight_to_token_ratio=0.149256 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.327509064e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7203601408 prefetch_bytes=1321205760 reduce_bytes=110100480 weight_movement_bytes=1431306240 egress_bytes=3602928640 ingress_bytes=5031979008 link_bytes=5031979008 total_nvlink_bytes=8634907648 weight_to_token_ratio=0.198693 prefetch_share_of_ingress=0.262562 dedup_ratio=0.114777 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=1 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=10160 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.751439113e-05 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60334 payload_rows_remote=52142 dup_rows=5202 dup_rows_remote=5202 dedup_ratio=0.079376 router_tokens_per_expert=[0, 0, 75, 0, 4086, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 33, 0, 0, 1, 2601, 0, 41, 9, 0, 0, 0, 0, 0, 3893, 4096, 0, 26, 1454, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040] payload_rows_per_dst=[8180, 8192, 8192, 8192, 5352, 8022, 6124, 8080] dup_rows_per_dst=[142, 0, 0, 0, 0, 3072, 1988, 0] combine_rows_per_src=[8180, 8192, 8192, 8192, 5352, 8022, 6124, 8080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=379488256 dispatch_meta_bytes=211792 combine_payload_bytes=379488256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=379700048 ingress_bytes=489588736 link_bytes=489588736 token_bytes=759188304 weight_movement_bytes=110100480 total_nvlink_bytes=869288784 weight_to_token_ratio=0.145024 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=18944 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.514753354e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6709059128 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=3355610680 ingress_bytes=5335257088 link_bytes=5335257088 total_nvlink_bytes=8690867768 weight_to_token_ratio=0.295393 prefetch_share_of_ingress=0.185728 dedup_ratio=0.167392 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037868 gemm_inner_pad_upper_bound=10795 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.278889575e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7163294888 prefetch_bytes=880803840 reduce_bytes=1541406720 weight_movement_bytes=2422210560 egress_bytes=3582778536 ingress_bytes=6002726912 link_bytes=6002726912 total_nvlink_bytes=9585505448 weight_to_token_ratio=0.338142 prefetch_share_of_ingress=0.146734 dedup_ratio=0.118152 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=14 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=9398 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.043972938e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=373753856 dispatch_meta_bytes=229376 combine_payload_bytes=373753856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373983232 ingress_bytes=483854336 link_bytes=483854336 token_bytes=747737088 weight_movement_bytes=110100480 total_nvlink_bytes=857837568 weight_to_token_ratio=0.147245 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.415022707e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7020603488 prefetch_bytes=1211105280 reduce_bytes=660602880 weight_movement_bytes=1871708160 egress_bytes=3511423072 ingress_bytes=5380888576 link_bytes=5380888576 total_nvlink_bytes=8892311648 weight_to_token_ratio=0.266602 prefetch_share_of_ingress=0.225075 dedup_ratio=0.140042 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=6 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=10541 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.358250238e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59882 payload_rows_remote=51690 dup_rows=5654 dup_rows_remote=5640 dedup_ratio=0.086273 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 1, 2985, 0, 39, 0, 0, 0, 0, 0, 0, 3867, 4096, 0, 0, 1474, 0, 98, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3798] payload_rows_per_dst=[8192, 8182, 8190, 8192, 8192, 8192, 3144, 7598] dup_rows_per_dst=[14, 0, 0, 0, 5640, 0, 0, 0] combine_rows_per_src=[8192, 8182, 8190, 8192, 8192, 8192, 3144, 7598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60092 payload_rows_remote=51900 dup_rows=5444 dup_rows_remote=5444 dedup_ratio=0.083069 router_tokens_per_expert=[0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 75, 0, 0, 0, 2354, 0, 40, 1, 0, 0, 0, 0, 0, 3838, 4095, 0, 0, 1967, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7176, 8192, 4202, 7754] dup_rows_per_dst=[2, 0, 0, 0, 3206, 2232, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7176, 8192, 4202, 7754] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7392038256 prefetch_bytes=990904320 reduce_bytes=1321205760 weight_movement_bytes=2312110080 egress_bytes=3697177968 ingress_bytes=6006970368 link_bytes=6006970368 total_nvlink_bytes=9704148336 weight_to_token_ratio=0.312784 prefetch_share_of_ingress=0.164959 dedup_ratio=0.108609 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=12 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=9017 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.044710945e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=370513920 dispatch_meta_bytes=229320 combine_payload_bytes=370513920 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=370743240 ingress_bytes=810915840 link_bytes=810915840 token_bytes=741257160 weight_movement_bytes=440401920 total_nvlink_bytes=1181659080 weight_to_token_ratio=0.594128 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.410316018e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=372019200 dispatch_meta_bytes=229376 combine_payload_bytes=372019200 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=372248576 ingress_bytes=482119680 link_bytes=482119680 token_bytes=744267776 weight_movement_bytes=110100480 total_nvlink_bytes=854368256 weight_to_token_ratio=0.147931 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.384854186e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6803821048 prefetch_bytes=990904320 reduce_bytes=2091909120 weight_movement_bytes=3082813440 egress_bytes=3403006456 ingress_bytes=6483628032 link_bytes=6483628032 total_nvlink_bytes=9886634488 weight_to_token_ratio=0.453100 prefetch_share_of_ingress=0.152832 dedup_ratio=0.156268 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=19 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=9779 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.127609552e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7196477760 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=3599360320 ingress_bytes=5358725120 link_bytes=5358725120 total_nvlink_bytes=8958085440 weight_to_token_ratio=0.244787 prefetch_share_of_ingress=0.164368 dedup_ratio=0.115482 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=9652 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.319704343e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none + [2026-08-26 18:09:52] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 370.2 | throughput per GPU (TFLOP/s/GPU): 333.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.944262E+00 | loss scale: 1.0 | grad norm: 23.055 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46382 payload_rows_remote=41242 dup_rows=19154 dup_rows_remote=17154 dedup_ratio=0.292267 router_tokens_per_expert=[108, 53, 82, 79, 6, 34, 796, 1170, 279, 98, 312, 1791, 1, 89, 0, 310, 325, 491, 2016, 11, 594, 351, 3, 214, 743, 218, 516, 761, 1, 1213, 892, 16, 27, 12, 103, 1909, 5, 14, 3, 706, 1005, 660, 639, 24, 2486, 509, 1082, 733, 33, 1402, 1119, 191, 668, 193, 70, 325, 1205, 1268, 405, 191, 335, 244, 1623, 6] payload_rows_per_dst=[4442, 7126, 5796, 5630, 5706, 5140, 5766, 6776] dup_rows_per_dst=[1260, 3606, 2214, 2044, 2016, 2000, 2236, 3778] combine_rows_per_src=[4442, 7126, 5796, 5630, 5706, 5140, 5766, 6776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45166 payload_rows_remote=38620 dup_rows=20370 dup_rows_remote=16114 dedup_ratio=0.310822 router_tokens_per_expert=[127, 31, 57, 78, 11, 34, 1341, 1392, 397, 15, 155, 1296, 1, 88, 0, 216, 309, 537, 1635, 2, 517, 432, 8, 253, 1001, 358, 283, 971, 3, 1046, 892, 0, 5, 10, 199, 2254, 0, 0, 25, 543, 1163, 386, 392, 4, 2405, 615, 1042, 1098, 78, 1457, 1040, 145, 1296, 428, 47, 289, 1129, 1045, 173, 182, 686, 143, 1001, 2] payload_rows_per_dst=[4728, 3792, 5376, 5832, 5840, 7070, 5982, 6546] dup_rows_per_dst=[1414, 544, 2010, 2170, 2316, 5056, 2604, 4256] combine_rows_per_src=[4728, 3792, 5376, 5832, 5840, 7070, 5982, 6546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46804 payload_rows_remote=40970 dup_rows=18732 dup_rows_remote=16670 dedup_ratio=0.285828 router_tokens_per_expert=[97, 71, 49, 37, 0, 10, 1104, 1671, 356, 104, 187, 744, 0, 139, 0, 415, 637, 540, 1829, 0, 156, 481, 6, 138, 922, 111, 505, 523, 5, 1844, 1028, 5, 12, 13, 87, 1655, 3, 0, 3, 345, 908, 302, 1313, 8, 2692, 655, 1787, 598, 149, 846, 1344, 182, 500, 309, 48, 134, 1271, 1153, 305, 860, 325, 204, 1043, 0] payload_rows_per_dst=[6284, 6848, 5558, 4890, 5834, 5532, 5262, 6596] dup_rows_per_dst=[3482, 2426, 2016, 1308, 2062, 1950, 1762, 3726] combine_rows_per_src=[6284, 6848, 5558, 4890, 5834, 5532, 5262, 6596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44438 payload_rows_remote=39188 dup_rows=21098 dup_rows_remote=19230 dedup_ratio=0.321930 router_tokens_per_expert=[134, 16, 33, 74, 31, 10, 823, 2442, 191, 30, 142, 1036, 2, 114, 1, 236, 127, 811, 1244, 0, 174, 391, 11, 239, 1416, 208, 283, 908, 2, 1882, 1330, 21, 33, 3, 53, 1733, 7, 3, 5, 404, 938, 671, 385, 8, 2415, 739, 715, 1047, 51, 785, 1244, 461, 675, 198, 70, 287, 926, 1628, 420, 422, 460, 478, 1140, 2] payload_rows_per_dst=[5618, 3240, 4688, 7110, 4750, 6972, 5250, 6810] dup_rows_per_dst=[1508, 404, 1306, 4990, 1162, 5294, 1868, 4566] combine_rows_per_src=[5618, 3240, 4688, 7110, 4750, 6972, 5250, 6810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45480 payload_rows_remote=39348 dup_rows=20056 dup_rows_remote=17418 dedup_ratio=0.306030 router_tokens_per_expert=[189, 39, 174, 63, 83, 5, 963, 1287, 440, 80, 109, 1165, 1, 190, 0, 171, 210, 834, 1854, 4, 262, 597, 2, 520, 1479, 575, 363, 737, 9, 2099, 1213, 9, 6, 1, 116, 1496, 4, 4, 38, 448, 380, 669, 500, 50, 2051, 639, 1235, 1251, 14, 1760, 1422, 166, 1310, 158, 42, 259, 555, 428, 227, 271, 480, 214, 848, 0] payload_rows_per_dst=[6212, 6178, 5980, 6132, 6202, 4038, 6424, 4314] dup_rows_per_dst=[3592, 2236, 2586, 2638, 2996, 1216, 3838, 954] combine_rows_per_src=[6212, 6178, 5980, 6132, 6202, 4038, 6424, 4314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45622 payload_rows_remote=38698 dup_rows=19914 dup_rows_remote=15814 dedup_ratio=0.303864 router_tokens_per_expert=[187, 58, 171, 104, 43, 10, 824, 1439, 1301, 239, 190, 1237, 0, 87, 1, 289, 176, 1496, 1640, 2, 315, 359, 8, 681, 1652, 254, 460, 661, 36, 1226, 792, 4, 45, 24, 173, 1488, 8, 2, 0, 455, 567, 746, 730, 31, 2168, 568, 892, 1298, 15, 946, 1107, 369, 1057, 184, 119, 35, 1106, 481, 169, 372, 524, 206, 940, 1] payload_rows_per_dst=[5628, 6924, 6496, 5728, 5970, 5274, 5604, 3998] dup_rows_per_dst=[2496, 4100, 3820, 1990, 2800, 1890, 2060, 758] combine_rows_per_src=[5628, 6924, 6496, 5728, 5970, 5274, 5604, 3998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46708 payload_rows_remote=41192 dup_rows=18828 dup_rows_remote=16698 dedup_ratio=0.287292 router_tokens_per_expert=[123, 121, 48, 155, 3, 19, 1204, 1772, 305, 22, 158, 846, 0, 66, 1, 189, 280, 695, 1522, 0, 267, 267, 2, 371, 1095, 88, 353, 799, 6, 1588, 903, 0, 10, 4, 112, 2025, 9, 0, 7, 253, 1213, 426, 562, 6, 2841, 774, 1410, 920, 34, 1028, 1466, 438, 534, 156, 40, 170, 1322, 1241, 423, 511, 405, 129, 1031, 0] payload_rows_per_dst=[6304, 6896, 5516, 5086, 6284, 5618, 5534, 5470] dup_rows_per_dst=[3762, 1960, 2130, 1402, 3216, 2406, 2198, 1754] combine_rows_per_src=[6304, 6896, 5516, 5086, 6284, 5618, 5534, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=295622656 dispatch_meta_bytes=233584 combine_payload_bytes=295622656 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=295856240 ingress_bytes=736024576 link_bytes=736024576 token_bytes=591478896 weight_movement_bytes=440401920 total_nvlink_bytes=1031880816 weight_to_token_ratio=0.744578 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.280067793e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276828160 dispatch_meta_bytes=218936 combine_payload_bytes=276828160 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277047096 ingress_bytes=607129600 link_bytes=607129600 token_bytes=553875256 weight_movement_bytes=330301440 total_nvlink_bytes=884176696 weight_to_token_ratio=0.596346 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055898230e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45890 payload_rows_remote=40202 dup_rows=19646 dup_rows_remote=16528 dedup_ratio=0.299774 router_tokens_per_expert=[109, 40, 112, 73, 2, 35, 671, 1499, 149, 123, 194, 646, 1, 45, 40, 405, 244, 555, 1460, 60, 424, 754, 11, 363, 1420, 190, 359, 1163, 7, 1862, 1141, 13, 65, 18, 65, 1177, 23, 0, 11, 995, 607, 526, 535, 1, 3025, 541, 901, 936, 327, 1478, 1296, 307, 446, 285, 89, 346, 782, 1151, 433, 737, 427, 183, 830, 55] payload_rows_per_dst=[5688, 6928, 6458, 6178, 5750, 4500, 6184, 4204] dup_rows_per_dst=[3118, 2328, 3586, 2408, 2632, 1580, 2964, 1030] combine_rows_per_src=[5688, 6928, 6458, 6178, 5750, 4500, 6184, 4204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=293672960 dispatch_meta_bytes=230560 combine_payload_bytes=293672960 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=293903520 ingress_bytes=513873920 link_bytes=513873920 token_bytes=587576480 weight_movement_bytes=220200960 total_nvlink_bytes=807777440 weight_to_token_ratio=0.374761 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=13824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.937112646e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280899584 dispatch_meta_bytes=233672 combine_payload_bytes=280899584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=281133256 ingress_bytes=501100544 link_bytes=501100544 token_bytes=562032840 weight_movement_bytes=220200960 total_nvlink_bytes=782233800 weight_to_token_ratio=0.391794 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.714962629e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282046464 dispatch_meta_bytes=227064 combine_payload_bytes=282046464 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=282273528 ingress_bytes=502247424 link_bytes=502247424 token_bytes=564319992 weight_movement_bytes=220200960 total_nvlink_bytes=784520952 weight_to_token_ratio=0.390206 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=9216 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.734908758e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277387264 dispatch_meta_bytes=218048 combine_payload_bytes=277387264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277605312 ingress_bytes=387487744 link_bytes=387487744 token_bytes=554992576 weight_movement_bytes=110100480 total_nvlink_bytes=665093056 weight_to_token_ratio=0.198382 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.739049176e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=295264256 dispatch_meta_bytes=231560 combine_payload_bytes=295264256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=295495816 ingress_bytes=405364736 link_bytes=405364736 token_bytes=590760072 weight_movement_bytes=110100480 total_nvlink_bytes=700860552 weight_to_token_ratio=0.186371 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.049959469e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=288167936 dispatch_meta_bytes=226920 combine_payload_bytes=288167936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=288394856 ingress_bytes=398268416 link_bytes=398268416 token_bytes=576562792 weight_movement_bytes=110100480 total_nvlink_bytes=686663272 weight_to_token_ratio=0.190960 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.926542793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52002 payload_rows_remote=44948 dup_rows=13534 dup_rows_remote=10932 dedup_ratio=0.206512 router_tokens_per_expert=[0, 3197, 0, 608, 0, 4076, 15, 44, 7, 0, 0, 0, 0, 2, 0, 0, 0, 1660, 3729, 65, 1, 3126, 0, 2932, 7, 3966, 271, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 115, 165, 3, 2, 0, 206, 0, 0, 0, 0, 0, 0, 3871, 469, 2922, 0, 0, 625, 57, 7, 0, 0, 1, 606, 9] payload_rows_per_dst=[3976, 7472, 7892, 7724, 8176, 2130, 7578, 7054] dup_rows_per_dst=[480, 4, 3902, 528, 214, 196, 5608, 2602] combine_rows_per_src=[3976, 7472, 7892, 7724, 8176, 2130, 7578, 7054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58376 payload_rows_remote=50696 dup_rows=7160 dup_rows_remote=6280 dedup_ratio=0.109253 router_tokens_per_expert=[0, 3342, 3, 592, 0, 4032, 29, 22, 0, 2, 1, 0, 25, 0, 0, 0, 0, 1264, 3856, 61, 0, 2581, 0, 2939, 71, 3877, 469, 0, 0, 0, 6, 0, 0, 4, 0, 0, 0, 0, 0, 516, 339, 0, 3, 0, 197, 0, 0, 0, 0, 0, 0, 3741, 621, 2978, 0, 14, 89, 195, 47, 2, 13, 0, 834, 3] payload_rows_per_dst=[7034, 7724, 6458, 7892, 8106, 7680, 6584, 6898] dup_rows_per_dst=[942, 44, 1354, 954, 998, 880, 642, 1346] combine_rows_per_src=[7034, 7724, 6458, 7892, 8106, 7680, 6584, 6898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58950 payload_rows_remote=50762 dup_rows=6586 dup_rows_remote=6108 dedup_ratio=0.100494 router_tokens_per_expert=[1, 3543, 0, 513, 0, 4092, 47, 45, 0, 12, 3, 0, 0, 0, 0, 1, 0, 1951, 3718, 118, 0, 1680, 0, 3047, 0, 3917, 202, 2, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 239, 433, 0, 5, 0, 145, 0, 0, 0, 0, 0, 0, 3872, 797, 3237, 1, 0, 94, 141, 12, 0, 0, 0, 888, 9] payload_rows_per_dst=[7336, 7438, 6184, 7864, 8188, 7896, 7076, 6968] dup_rows_per_dst=[962, 30, 1314, 380, 478, 1014, 994, 1414] combine_rows_per_src=[7336, 7438, 6184, 7864, 8188, 7896, 7076, 6968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57578 payload_rows_remote=52258 dup_rows=7958 dup_rows_remote=7386 dedup_ratio=0.121429 router_tokens_per_expert=[0, 3508, 0, 751, 1, 4088, 55, 31, 0, 28, 11, 0, 0, 0, 2, 1, 0, 1500, 3496, 68, 0, 2967, 0, 3132, 34, 3873, 194, 0, 4, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 141, 946, 2, 11, 0, 205, 2, 1, 0, 0, 8, 1, 3814, 542, 2387, 8, 0, 123, 10, 12, 0, 0, 1, 801, 4] payload_rows_per_dst=[7318, 7030, 7140, 7778, 8180, 8002, 5320, 6810] dup_rows_per_dst=[1374, 46, 1930, 432, 288, 1960, 572, 1356] combine_rows_per_src=[7318, 7030, 7140, 7778, 8180, 8002, 5320, 6810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57388 payload_rows_remote=49480 dup_rows=8148 dup_rows_remote=7434 dedup_ratio=0.124329 router_tokens_per_expert=[0, 3267, 0, 564, 0, 4078, 33, 7, 0, 0, 26, 0, 0, 0, 0, 0, 0, 2046, 3536, 118, 0, 2548, 0, 2888, 34, 3922, 354, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 341, 670, 0, 2, 0, 643, 0, 0, 0, 0, 1, 0, 3375, 753, 2992, 0, 0, 60, 154, 12, 0, 0, 0, 343, 0] payload_rows_per_dst=[6810, 7086, 7042, 7908, 8162, 7356, 6694, 6330] dup_rows_per_dst=[932, 38, 2382, 714, 676, 2024, 798, 584] combine_rows_per_src=[6810, 7086, 7042, 7908, 8162, 7356, 6694, 6330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57408 payload_rows_remote=49948 dup_rows=8128 dup_rows_remote=6802 dedup_ratio=0.124023 router_tokens_per_expert=[0, 3541, 0, 395, 0, 4095, 60, 15, 0, 13, 3, 0, 0, 0, 0, 0, 0, 1437, 3655, 70, 0, 2755, 0, 2631, 5, 3995, 223, 0, 0, 0, 1, 0, 0, 15, 0, 0, 0, 0, 0, 475, 378, 0, 18, 1, 350, 0, 1, 0, 0, 0, 1, 3793, 961, 2867, 0, 0, 357, 167, 63, 0, 0, 0, 423, 4] payload_rows_per_dst=[7232, 7460, 5886, 8000, 8190, 7722, 6696, 6222] dup_rows_per_dst=[790, 1326, 1194, 448, 980, 1360, 962, 1068] combine_rows_per_src=[7232, 7460, 5886, 8000, 8190, 7722, 6696, 6222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=322187264 dispatch_meta_bytes=223520 combine_payload_bytes=322187264 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=322410784 ingress_bytes=542388224 link_bytes=542388224 token_bytes=644598048 weight_movement_bytes=220200960 total_nvlink_bytes=864799008 weight_to_token_ratio=0.341610 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=23296 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.433023291e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=363388928 dispatch_meta_bytes=227904 combine_payload_bytes=363388928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=363616832 ingress_bytes=473489408 link_bytes=473489408 token_bytes=727005760 weight_movement_bytes=110100480 total_nvlink_bytes=837106240 weight_to_token_ratio=0.151444 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=26624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.234759561e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58874 payload_rows_remote=52016 dup_rows=6662 dup_rows_remote=4984 dedup_ratio=0.101654 router_tokens_per_expert=[0, 3613, 5, 688, 0, 4092, 30, 50, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1223, 3901, 110, 0, 2930, 5, 2909, 8, 3582, 249, 0, 2, 1, 0, 0, 2, 3, 0, 0, 2, 1, 0, 144, 213, 2, 0, 0, 207, 0, 0, 0, 0, 0, 3, 3898, 637, 3071, 0, 0, 137, 43, 30, 0, 1, 0, 956, 17] payload_rows_per_dst=[7474, 7804, 6858, 7232, 8192, 7902, 6668, 6744] dup_rows_per_dst=[1298, 4, 1678, 452, 296, 738, 754, 1442] combine_rows_per_src=[7474, 7804, 6858, 7232, 8192, 7902, 6668, 6744] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=363862016 dispatch_meta_bytes=227480 combine_payload_bytes=363862016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364089496 ingress_bytes=473962496 link_bytes=473962496 token_bytes=727951512 weight_movement_bytes=110100480 total_nvlink_bytes=838051992 weight_to_token_ratio=0.151247 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.242987340e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=374585344 dispatch_meta_bytes=238576 combine_payload_bytes=374585344 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=374823920 ingress_bytes=594786304 link_bytes=594786304 token_bytes=749409264 weight_movement_bytes=220200960 total_nvlink_bytes=969610224 weight_to_token_ratio=0.293833 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=24064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.034431208e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=354672640 dispatch_meta_bytes=227656 combine_payload_bytes=354672640 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=354900296 ingress_bytes=574873600 link_bytes=574873600 token_bytes=709572936 weight_movement_bytes=220200960 total_nvlink_bytes=929773896 weight_to_token_ratio=0.310329 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.997997409e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=358027264 dispatch_meta_bytes=227000 combine_payload_bytes=358027264 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=358254264 ingress_bytes=578228224 link_bytes=578228224 token_bytes=716281528 weight_movement_bytes=220200960 total_nvlink_bytes=936482488 weight_to_token_ratio=0.307422 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.005633984e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51894 payload_rows_remote=44418 dup_rows=13642 dup_rows_remote=9540 dedup_ratio=0.208160 router_tokens_per_expert=[0, 2766, 0, 511, 20, 3798, 97, 146, 1, 3, 13, 0, 47, 0, 1, 2, 1, 1997, 3479, 123, 40, 2469, 0, 2691, 26, 3475, 278, 4, 4, 0, 14, 0, 0, 65, 0, 1, 0, 3, 0, 402, 335, 5, 7, 4, 679, 1, 1, 14, 2, 6, 2, 3292, 783, 3496, 0, 6, 637, 186, 7, 1, 0, 13, 650, 164] payload_rows_per_dst=[7476, 7908, 4154, 6850, 4090, 7250, 7390, 6776] dup_rows_per_dst=[4102, 4122, 168, 380, 322, 1426, 1200, 1922] combine_rows_per_src=[7476, 7908, 4154, 6850, 4090, 7250, 7390, 6776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=372850688 dispatch_meta_bytes=228000 combine_payload_bytes=372850688 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=373078688 ingress_bytes=923353088 link_bytes=923353088 token_bytes=745929376 weight_movement_bytes=550502400 total_nvlink_bytes=1296431776 weight_to_token_ratio=0.738009 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.605862886e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51404 payload_rows_remote=43386 dup_rows=14132 dup_rows_remote=13536 dedup_ratio=0.215637 router_tokens_per_expert=[0, 1, 0, 0, 64, 4, 1, 1, 1, 3, 1, 4094, 3858, 4003, 334, 0, 0, 0, 0, 0, 4071, 1, 1, 0, 0, 1243, 0, 23, 1, 2, 0, 0, 0, 396, 0, 3928, 0, 0, 20, 5, 1, 1, 1, 0, 4008, 0, 1873, 0, 0, 0, 0, 0, 4025, 3, 496, 0, 0, 0, 0, 0, 303, 0, 0, 1] payload_rows_per_dst=[4866, 3736, 8144, 2496, 7896, 8190, 8058, 8018] dup_rows_per_dst=[94, 312, 2, 42, 802, 11294, 990, 596] combine_rows_per_src=[4866, 3736, 8144, 2496, 7896, 8190, 8058, 8018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=318388224 dispatch_meta_bytes=215832 combine_payload_bytes=318388224 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=318604056 ingress_bytes=648689664 link_bytes=648689664 token_bytes=636992280 weight_movement_bytes=330301440 total_nvlink_bytes=967293720 weight_to_token_ratio=0.518533 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=25216 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.128178017e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56422 payload_rows_remote=48322 dup_rows=9114 dup_rows_remote=7330 dedup_ratio=0.139069 router_tokens_per_expert=[0, 0, 0, 13, 483, 0, 0, 0, 0, 0, 0, 4096, 3838, 3880, 607, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 1377, 0, 1, 0, 0, 0, 0, 0, 1227, 0, 3566, 0, 7, 110, 14, 0, 49, 0, 0, 3681, 0, 1055, 0, 0, 0, 0, 0, 3773, 0, 532, 0, 0, 0, 23, 0, 360, 0, 0, 0] payload_rows_per_dst=[8192, 1214, 8152, 7592, 7726, 8100, 7628, 7818] dup_rows_per_dst=[992, 0, 0, 2526, 2122, 1784, 982, 708] combine_rows_per_src=[8192, 1214, 8152, 7592, 7726, 8100, 7628, 7818] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57838 payload_rows_remote=50598 dup_rows=7698 dup_rows_remote=6862 dedup_ratio=0.117462 router_tokens_per_expert=[0, 2, 0, 0, 95, 0, 0, 0, 2, 1, 0, 4094, 3822, 4043, 382, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 1168, 1, 0, 0, 0, 1, 0, 0, 564, 0, 3463, 0, 2, 7, 2, 0, 2, 0, 1, 3902, 0, 1630, 0, 0, 0, 0, 1, 3904, 0, 537, 0, 0, 0, 2, 0, 1095, 0, 0, 2] payload_rows_per_dst=[8192, 4924, 8086, 8000, 7240, 5430, 7830, 8136] dup_rows_per_dst=[190, 190, 0, 2144, 836, 1136, 1054, 2148] combine_rows_per_src=[8192, 4924, 8086, 8000, 7240, 5430, 7830, 8136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54044 payload_rows_remote=45890 dup_rows=11492 dup_rows_remote=5496 dedup_ratio=0.175354 router_tokens_per_expert=[0, 0, 0, 1, 284, 0, 0, 0, 0, 0, 0, 4095, 3892, 3885, 414, 0, 0, 0, 0, 0, 3959, 0, 0, 0, 0, 1074, 0, 3, 0, 0, 20, 0, 1, 967, 0, 3647, 0, 0, 6, 0, 0, 0, 0, 0, 3653, 0, 1568, 0, 0, 0, 0, 0, 3918, 0, 621, 0, 0, 0, 0, 0, 756, 0, 0, 4] payload_rows_per_dst=[8192, 8154, 7918, 7604, 3252, 3136, 7856, 7932] dup_rows_per_dst=[568, 5996, 0, 1896, 452, 0, 1222, 1358] combine_rows_per_src=[8192, 8154, 7918, 7604, 3252, 3136, 7856, 7932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56770 payload_rows_remote=49526 dup_rows=8766 dup_rows_remote=6902 dedup_ratio=0.133759 router_tokens_per_expert=[0, 0, 0, 1, 295, 0, 0, 0, 0, 0, 0, 4096, 3890, 3863, 947, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 1188, 0, 0, 0, 0, 2, 0, 0, 1949, 0, 3671, 0, 0, 6, 0, 0, 0, 0, 0, 3364, 0, 817, 0, 0, 0, 0, 0, 3927, 0, 408, 0, 0, 0, 0, 0, 313, 0, 0, 4] payload_rows_per_dst=[8192, 8120, 8054, 7244, 7912, 1634, 7868, 7746] dup_rows_per_dst=[592, 1554, 0, 1864, 3340, 0, 802, 614] combine_rows_per_src=[8192, 8120, 8054, 7244, 7912, 1634, 7868, 7746] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52282 payload_rows_remote=44926 dup_rows=13254 dup_rows_remote=12286 dedup_ratio=0.202240 router_tokens_per_expert=[1, 0, 0, 1, 168, 0, 0, 0, 13, 4, 0, 4091, 3861, 3980, 444, 0, 0, 1, 0, 0, 4051, 0, 1, 0, 0, 1638, 0, 0, 1, 0, 2, 0, 1, 779, 1, 3709, 0, 0, 13, 0, 0, 15, 7, 0, 3780, 0, 1555, 0, 0, 0, 0, 0, 3622, 0, 540, 0, 2, 0, 0, 0, 455, 3, 0, 29] payload_rows_per_dst=[8186, 922, 8106, 3828, 7704, 8180, 7356, 8000] dup_rows_per_dst=[336, 0, 0, 422, 1302, 9288, 968, 938] combine_rows_per_src=[8186, 922, 8106, 3828, 7704, 8180, 7356, 8000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=310990848 dispatch_meta_bytes=227688 combine_payload_bytes=310990848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=311218536 ingress_bytes=421091328 link_bytes=421091328 token_bytes=622209384 weight_movement_bytes=110100480 total_nvlink_bytes=732309864 weight_to_token_ratio=0.176951 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.323470770e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=346372096 dispatch_meta_bytes=222608 combine_payload_bytes=346372096 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=346594704 ingress_bytes=566573056 link_bytes=566573056 token_bytes=692966800 weight_movement_bytes=220200960 total_nvlink_bytes=913167760 weight_to_token_ratio=0.317766 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=13312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.853637296e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58310 payload_rows_remote=50140 dup_rows=7226 dup_rows_remote=7224 dedup_ratio=0.110260 router_tokens_per_expert=[0, 2, 2, 0, 121, 0, 0, 0, 1, 0, 0, 4092, 3837, 4049, 381, 3, 0, 4, 0, 0, 4081, 1, 0, 0, 4, 1113, 0, 0, 1, 0, 0, 1, 0, 842, 0, 3629, 0, 1, 12, 0, 1, 9, 0, 1, 3960, 0, 1129, 0, 3, 1, 0, 0, 3951, 0, 777, 0, 1, 1, 0, 1, 739, 0, 0, 17] payload_rows_per_dst=[8192, 8032, 8170, 8002, 7556, 2272, 7940, 8146] dup_rows_per_dst=[242, 412, 2, 2156, 1412, 8, 1524, 1470] combine_rows_per_src=[8192, 8032, 8170, 8002, 7556, 2272, 7940, 8146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=328939520 dispatch_meta_bytes=205544 combine_payload_bytes=328939520 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=329145064 ingress_bytes=769341440 link_bytes=769341440 token_bytes=658084584 weight_movement_bytes=440401920 total_nvlink_bytes=1098486504 weight_to_token_ratio=0.669218 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.338011299e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=362686464 dispatch_meta_bytes=229840 combine_payload_bytes=362686464 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=362916304 ingress_bytes=582887424 link_bytes=582887424 token_bytes=725602768 weight_movement_bytes=220200960 total_nvlink_bytes=945803728 weight_to_token_ratio=0.303473 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.013737099e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=355002368 dispatch_meta_bytes=225712 combine_payload_bytes=355002368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=355228080 ingress_bytes=465102848 link_bytes=465102848 token_bytes=710230448 weight_movement_bytes=110100480 total_nvlink_bytes=820330928 weight_to_token_ratio=0.155021 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.088903489e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=322029568 dispatch_meta_bytes=228848 combine_payload_bytes=322029568 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=322258416 ingress_bytes=542230528 link_bytes=542230528 token_bytes=644287984 weight_movement_bytes=220200960 total_nvlink_bytes=864488944 weight_to_token_ratio=0.341774 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.430280698e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47178 payload_rows_remote=39016 dup_rows=18358 dup_rows_remote=18010 dedup_ratio=0.280121 router_tokens_per_expert=[0, 0, 0, 50, 119, 43, 1, 0, 172, 5, 0, 4042, 3680, 4016, 322, 0, 0, 0, 0, 0, 3729, 0, 2, 2, 0, 1169, 4, 4, 0, 0, 61, 0, 0, 665, 2, 3446, 0, 0, 10, 66, 3, 14, 0, 0, 3622, 0, 2525, 0, 0, 0, 18, 0, 3777, 1, 900, 0, 0, 0, 2, 16, 252, 27, 1, 0] payload_rows_per_dst=[8162, 8192, 7872, 7812, 5128, 5066, 3170, 1776] dup_rows_per_dst=[348, 13778, 1030, 1908, 856, 18, 288, 132] combine_rows_per_src=[8162, 8192, 7872, 7812, 5128, 5066, 3170, 1776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=359403520 dispatch_meta_bytes=229456 combine_payload_bytes=359403520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=359632976 ingress_bytes=469504000 link_bytes=469504000 token_bytes=719036496 weight_movement_bytes=110100480 total_nvlink_bytes=829136976 weight_to_token_ratio=0.153122 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.165446761e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62876 payload_rows_remote=54714 dup_rows=2660 dup_rows_remote=2368 dedup_ratio=0.040588 router_tokens_per_expert=[0, 27, 0, 0, 0, 0, 0, 1, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4080, 3380, 0, 4096, 1, 1, 0, 0, 4094, 0, 4095, 47, 0, 0, 1, 1, 0, 1, 0, 0, 0, 712, 4001, 1, 1, 4093, 0, 0, 0, 0, 0, 0, 2, 0, 4077, 4, 0, 0, 0, 0, 0] payload_rows_per_dst=[7864, 7584, 8164, 6874, 7898, 8142, 8188, 8162] dup_rows_per_dst=[50, 100, 0, 828, 94, 1290, 6, 292] combine_rows_per_src=[7864, 7584, 8164, 6874, 7898, 8142, 8188, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=228104 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279894792 ingress_bytes=389767168 link_bytes=389767168 token_bytes=559561480 weight_movement_bytes=110100480 total_nvlink_bytes=669661960 weight_to_token_ratio=0.196762 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778692108e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63036 payload_rows_remote=54848 dup_rows=2500 dup_rows_remote=114 dedup_ratio=0.038147 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4070, 2898, 0, 4096, 0, 0, 0, 6, 4096, 0, 4076, 4, 0, 0, 8, 0, 0, 0, 0, 1, 0, 1234, 4052, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8140, 5796, 8166, 8188, 8192, 8170] dup_rows_per_dst=[4, 86, 2, 12, 10, 2386, 0, 0] combine_rows_per_src=[8192, 8192, 8140, 5796, 8166, 8188, 8192, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61106 payload_rows_remote=52914 dup_rows=4430 dup_rows_remote=4430 dedup_ratio=0.067596 router_tokens_per_expert=[0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 3143, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 3949, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7282, 8190, 4678, 8192, 8188] dup_rows_per_dst=[10, 0, 10, 3502, 4, 900, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 7282, 8190, 4678, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=392189952 dispatch_meta_bytes=228328 combine_payload_bytes=392189952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=392418280 ingress_bytes=502290432 link_bytes=502290432 token_bytes=784608232 weight_movement_bytes=110100480 total_nvlink_bytes=894708712 weight_to_token_ratio=0.140325 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.735656738e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64594 payload_rows_remote=56402 dup_rows=942 dup_rows_remote=850 dedup_ratio=0.014374 router_tokens_per_expert=[0, 4, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3631, 0, 4095, 1, 0, 0, 0, 4095, 0, 4095, 46, 0, 0, 1, 0, 0, 2, 0, 0, 0, 521, 3992, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8186, 7262, 8192, 8192, 8192, 8188] dup_rows_per_dst=[8, 2, 0, 2, 92, 838, 0, 0] combine_rows_per_src=[8192, 8190, 8186, 7262, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64176 payload_rows_remote=57328 dup_rows=1360 dup_rows_remote=1360 dedup_ratio=0.020752 router_tokens_per_expert=[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4088, 3424, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 749, 3999, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8176, 6848, 8192, 8192, 8192, 8192] dup_rows_per_dst=[12, 0, 2, 0, 42, 1304, 0, 0] combine_rows_per_src=[8192, 8192, 8176, 6848, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63286 payload_rows_remote=55094 dup_rows=2250 dup_rows_remote=2246 dedup_ratio=0.034332 router_tokens_per_expert=[1, 3, 0, 1, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4083, 2991, 1, 4094, 1, 0, 3, 6, 4095, 0, 4095, 33, 0, 0, 0, 2, 2, 0, 0, 0, 0, 1094, 4066, 1, 0, 4095, 0, 0, 0, 2, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8166, 5984, 8192, 8186, 8192, 8186] dup_rows_per_dst=[12, 4, 2, 20, 72, 2136, 4, 0] combine_rows_per_src=[8192, 8188, 8166, 5984, 8192, 8186, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=393150464 dispatch_meta_bytes=219848 combine_payload_bytes=393150464 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=393370312 ingress_bytes=613351424 link_bytes=613351424 token_bytes=786520776 weight_movement_bytes=220200960 total_nvlink_bytes=1006721736 weight_to_token_ratio=0.279968 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=25856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.066719005e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64096 payload_rows_remote=55908 dup_rows=1440 dup_rows_remote=1438 dedup_ratio=0.021973 router_tokens_per_expert=[0, 22, 0, 2, 4, 0, 0, 0, 25, 1, 0, 4, 0, 1, 0, 0, 0, 1, 0, 2, 0, 0, 1, 4091, 3404, 0, 4091, 0, 0, 1, 0, 4094, 0, 4091, 36, 1, 3, 0, 0, 1, 4, 0, 1, 0, 747, 3959, 0, 0, 4091, 0, 0, 0, 0, 1, 2, 0, 2, 4084, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 6810, 8192, 8164, 8186, 8172] dup_rows_per_dst=[52, 52, 2, 0, 72, 1258, 2, 2] combine_rows_per_src=[8192, 8192, 8188, 6810, 8192, 8164, 8186, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=379287552 dispatch_meta_bytes=229376 combine_payload_bytes=379287552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=379516928 ingress_bytes=489388032 link_bytes=489388032 token_bytes=758804480 weight_movement_bytes=110100480 total_nvlink_bytes=868904960 weight_to_token_ratio=0.145097 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.511262781e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=404289536 dispatch_meta_bytes=229008 combine_payload_bytes=404289536 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=404518544 ingress_bytes=734590976 link_bytes=734590976 token_bytes=808808080 weight_movement_bytes=330301440 total_nvlink_bytes=1139109520 weight_to_token_ratio=0.408380 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.277574527e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=410927104 dispatch_meta_bytes=234752 combine_payload_bytes=410927104 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411161856 ingress_bytes=741228544 link_bytes=741228544 token_bytes=822088960 weight_movement_bytes=330301440 total_nvlink_bytes=1152390400 weight_to_token_ratio=0.401783 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289118349e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=394913792 dispatch_meta_bytes=229360 combine_payload_bytes=394913792 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=395143152 ingress_bytes=615114752 link_bytes=615114752 token_bytes=790056944 weight_movement_bytes=220200960 total_nvlink_bytes=1010257904 weight_to_token_ratio=0.278715 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.069785723e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58688 payload_rows_remote=50496 dup_rows=6848 dup_rows_remote=6668 dedup_ratio=0.104492 router_tokens_per_expert=[0, 86, 1, 3, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4094, 3271, 0, 4096, 13, 0, 0, 87, 4096, 0, 4096, 21, 0, 0, 0, 0, 0, 2, 0, 17, 1, 814, 3787, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 4, 2, 3990, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8086, 1680, 8176, 7984] dup_rows_per_dst=[180, 370, 114, 6130, 42, 2, 10, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 8086, 1680, 8176, 7984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=400748544 dispatch_meta_bytes=229384 combine_payload_bytes=400748544 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400977928 ingress_bytes=510849024 link_bytes=510849024 token_bytes=801726472 weight_movement_bytes=110100480 total_nvlink_bytes=911826952 weight_to_token_ratio=0.137329 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.884504730e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63496 payload_rows_remote=55354 dup_rows=2040 dup_rows_remote=1260 dedup_ratio=0.031128 router_tokens_per_expert=[0, 0, 0, 4094, 1, 0, 4095, 0, 0, 0, 2, 4091, 0, 0, 1, 0, 0, 7, 1, 2, 0, 1, 0, 1, 0, 4064, 0, 5, 4090, 0, 3825, 0, 0, 0, 276, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 19, 0, 0, 0, 0, 1, 1, 1] payload_rows_per_dst=[7544, 8184, 8192, 6874, 8188, 8188, 8184, 8142] dup_rows_per_dst=[0, 634, 20, 4, 552, 18, 32, 780] combine_rows_per_src=[7544, 8184, 8192, 6874, 8188, 8188, 8184, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=361955328 dispatch_meta_bytes=228656 combine_payload_bytes=361955328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362183984 ingress_bytes=472055808 link_bytes=472055808 token_bytes=724139312 weight_movement_bytes=110100480 total_nvlink_bytes=834239792 weight_to_token_ratio=0.152043 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.209826900e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65160 payload_rows_remote=56968 dup_rows=376 dup_rows_remote=376 dedup_ratio=0.005737 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 4, 0, 0, 13, 0, 0, 0, 4029, 0, 0, 4096, 0, 3981, 0, 0, 0, 166, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8168, 8192, 7962, 8192, 8192, 8192, 8070] dup_rows_per_dst=[0, 0, 34, 0, 332, 0, 0, 10] combine_rows_per_src=[8192, 8168, 8192, 7962, 8192, 8192, 8192, 8070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64726 payload_rows_remote=56540 dup_rows=810 dup_rows_remote=810 dedup_ratio=0.012360 router_tokens_per_expert=[0, 1, 30, 4096, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 0, 0, 0, 4027, 0, 0, 4096, 0, 3748, 0, 0, 0, 317, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 7496, 8190, 8192, 8192, 8086] dup_rows_per_dst=[62, 0, 10, 0, 634, 0, 0, 104] combine_rows_per_src=[8192, 8186, 8192, 7496, 8190, 8192, 8192, 8086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=396777472 dispatch_meta_bytes=226456 combine_payload_bytes=396777472 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=397003928 ingress_bytes=616978432 link_bytes=616978432 token_bytes=793781400 weight_movement_bytes=220200960 total_nvlink_bytes=1013982360 weight_to_token_ratio=0.277408 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.073026969e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64538 payload_rows_remote=56346 dup_rows=998 dup_rows_remote=20 dedup_ratio=0.015228 router_tokens_per_expert=[0, 1, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 1, 0, 0, 4025, 0, 0, 4093, 0, 3692, 0, 0, 0, 489, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8150, 8192, 7384, 8192, 8192, 8186, 8052] dup_rows_per_dst=[0, 0, 14, 0, 978, 2, 0, 4] combine_rows_per_src=[8190, 8150, 8192, 7384, 8192, 8192, 8186, 8052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64340 payload_rows_remote=57124 dup_rows=1196 dup_rows_remote=1196 dedup_ratio=0.018250 router_tokens_per_expert=[0, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 4095, 0, 3608, 0, 0, 0, 593, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 8192, 7216, 8192, 8190, 8190, 7984] dup_rows_per_dst=[2, 0, 2, 0, 1186, 0, 0, 6] combine_rows_per_src=[8192, 8184, 8192, 7216, 8192, 8190, 8190, 7984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64946 payload_rows_remote=56756 dup_rows=590 dup_rows_remote=590 dedup_ratio=0.009003 router_tokens_per_expert=[1, 1, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 10, 5, 1, 0, 0, 3, 0, 0, 4077, 0, 5, 4095, 0, 3819, 0, 0, 0, 249, 0, 0, 0, 4095, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1, 6, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 7644, 8190, 8192, 8190, 8166] dup_rows_per_dst=[2, 0, 36, 4, 498, 2, 0, 48] combine_rows_per_src=[8192, 8180, 8192, 7644, 8190, 8192, 8190, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=408346624 dispatch_meta_bytes=229376 combine_payload_bytes=408346624 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=408576000 ingress_bytes=628547584 link_bytes=628547584 token_bytes=816922624 weight_movement_bytes=220200960 total_nvlink_bytes=1037123584 weight_to_token_ratio=0.269549 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.093147627e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64226 payload_rows_remote=56036 dup_rows=1310 dup_rows_remote=1288 dedup_ratio=0.019989 router_tokens_per_expert=[0, 0, 2, 4093, 5, 0, 4094, 0, 0, 0, 0, 4072, 0, 3, 0, 3, 0, 1, 8, 4, 0, 0, 0, 0, 0, 3953, 0, 2, 4090, 0, 3606, 0, 0, 1, 619, 0, 0, 0, 4091, 0, 14, 0, 2, 4091, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 1, 1, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8154, 8190, 7214, 8184, 8192, 8186, 7914] dup_rows_per_dst=[10, 2, 22, 2, 1238, 26, 4, 6] combine_rows_per_src=[8192, 8154, 8190, 7214, 8184, 8192, 8186, 7914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=405278720 dispatch_meta_bytes=229400 combine_payload_bytes=405278720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405508120 ingress_bytes=515379200 link_bytes=515379200 token_bytes=810786840 weight_movement_bytes=110100480 total_nvlink_bytes=920887320 weight_to_token_ratio=0.135795 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.963291941e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58768 payload_rows_remote=50576 dup_rows=6768 dup_rows_remote=6194 dedup_ratio=0.103271 router_tokens_per_expert=[0, 55, 24, 4096, 0, 0, 4096, 0, 4, 0, 0, 3843, 0, 0, 0, 0, 0, 48, 0, 0, 0, 15, 77, 0, 0, 3861, 0, 13, 4090, 0, 3733, 0, 0, 0, 321, 0, 0, 0, 4096, 0, 102, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7694, 7992, 8064, 2566, 8126, 8180, 7954] dup_rows_per_dst=[574, 0, 64, 5672, 224, 24, 0, 210] combine_rows_per_src=[8192, 7694, 7992, 8064, 2566, 8126, 8180, 7954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=403888128 dispatch_meta_bytes=225464 combine_payload_bytes=403888128 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=404113592 ingress_bytes=624089088 link_bytes=624089088 token_bytes=808001720 weight_movement_bytes=220200960 total_nvlink_bytes=1028202680 weight_to_token_ratio=0.272525 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=29696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.085393569e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409464832 dispatch_meta_bytes=233280 combine_payload_bytes=409464832 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=409698112 ingress_bytes=959967232 link_bytes=959967232 token_bytes=819162944 weight_movement_bytes=550502400 total_nvlink_bytes=1369665344 weight_to_token_ratio=0.672030 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.669540904e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=406827008 dispatch_meta_bytes=229384 combine_payload_bytes=406827008 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=407056392 ingress_bytes=627027968 link_bytes=627027968 token_bytes=813883400 weight_movement_bytes=220200960 total_nvlink_bytes=1034084360 weight_to_token_ratio=0.270556 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.090504765e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58550 payload_rows_remote=50366 dup_rows=6986 dup_rows_remote=354 dedup_ratio=0.106598 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 1, 0, 1, 1, 0, 1, 2601, 0, 0, 0, 1, 4046, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 4094, 0, 0, 223, 0, 1, 0, 0, 0, 0, 1, 0, 2, 0, 56, 0, 0, 1, 0, 0, 5, 0, 0, 4040, 1342, 0, 0, 0, 0, 4094] payload_rows_per_dst=[8188, 8136, 5206, 8092, 4360, 8192, 8192, 8184] dup_rows_per_dst=[0, 2, 2, 0, 226, 2, 122, 6632] combine_rows_per_src=[8188, 8136, 5206, 8092, 4360, 8192, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=401666048 dispatch_meta_bytes=229296 combine_payload_bytes=401666048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401895344 ingress_bytes=511766528 link_bytes=511766528 token_bytes=803561392 weight_movement_bytes=110100480 total_nvlink_bytes=913661872 weight_to_token_ratio=0.137016 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.900461633e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=362528768 dispatch_meta_bytes=227080 combine_payload_bytes=362528768 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=362755848 ingress_bytes=692830208 link_bytes=692830208 token_bytes=725284616 weight_movement_bytes=330301440 total_nvlink_bytes=1055586056 weight_to_token_ratio=0.455409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.204945683e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60428 payload_rows_remote=52236 dup_rows=5108 dup_rows_remote=5108 dedup_ratio=0.077942 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1697, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 2017, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8188, 7986, 3394, 8092, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 1052, 0, 42, 4014] combine_rows_per_src=[8188, 7986, 3394, 8092, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59758 payload_rows_remote=51854 dup_rows=5778 dup_rows_remote=5778 dedup_ratio=0.088165 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2341, 0, 0, 0, 0, 3728, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 1721, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 7904, 8188, 7456, 8192, 8192, 8192, 3442] dup_rows_per_dst=[0, 0, 4682, 0, 1012, 0, 84, 0] combine_rows_per_src=[8192, 7904, 8188, 7456, 8192, 8192, 8192, 3442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=361023488 dispatch_meta_bytes=202880 combine_payload_bytes=361023488 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=361226368 ingress_bytes=691324928 link_bytes=691324928 token_bytes=722249856 weight_movement_bytes=330301440 total_nvlink_bytes=1052551296 weight_to_token_ratio=0.457323 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=15616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.202327753e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60444 payload_rows_remote=52252 dup_rows=5092 dup_rows_remote=4770 dedup_ratio=0.077698 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1541, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 2469, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8190, 8108, 4294, 8070, 8192, 8190, 8192, 7208] dup_rows_per_dst=[0, 0, 950, 0, 322, 0, 86, 3734] combine_rows_per_src=[8190, 8108, 4294, 8070, 8192, 8190, 8192, 7208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60480 payload_rows_remote=52290 dup_rows=5056 dup_rows_remote=4996 dedup_ratio=0.077148 router_tokens_per_expert=[0, 0, 4095, 1, 0, 0, 0, 1, 1, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1708, 1, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4095, 0, 0, 311, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 28, 0, 0, 0, 0, 0, 2, 0, 1, 4068, 2212, 0, 0, 0, 0, 4094] payload_rows_per_dst=[8192, 8074, 3420, 8032, 8190, 8190, 8190, 8192] dup_rows_per_dst=[2, 0, 0, 0, 622, 2, 60, 4370] combine_rows_per_src=[8192, 8074, 3420, 8032, 8190, 8190, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60256 payload_rows_remote=52226 dup_rows=5280 dup_rows_remote=5280 dedup_ratio=0.080566 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1472, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1671, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 7924, 8192, 8030, 8192, 8192, 8192, 3342] dup_rows_per_dst=[0, 0, 2944, 0, 2296, 0, 40, 0] combine_rows_per_src=[8192, 7924, 8192, 8030, 8192, 8192, 8192, 3342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=374427648 dispatch_meta_bytes=229376 combine_payload_bytes=374427648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374657024 ingress_bytes=484528128 link_bytes=484528128 token_bytes=749084672 weight_movement_bytes=110100480 total_nvlink_bytes=859185152 weight_to_token_ratio=0.146980 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.426741058e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=371689472 dispatch_meta_bytes=230528 combine_payload_bytes=371689472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=371920000 ingress_bytes=481789952 link_bytes=481789952 token_bytes=743609472 weight_movement_bytes=110100480 total_nvlink_bytes=853709952 weight_to_token_ratio=0.148062 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.379119674e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61520 payload_rows_remote=53332 dup_rows=4016 dup_rows_remote=884 dedup_ratio=0.061279 router_tokens_per_expert=[0, 0, 4091, 0, 0, 0, 0, 3, 0, 3994, 0, 1, 0, 1, 0, 0, 0, 0, 3, 1571, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 5, 4093, 0, 0, 0, 4093, 0, 3, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 1, 0, 0, 0, 2, 0, 4086, 2273, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8188, 7990, 8188, 8036, 8190, 8192, 8190, 4546] dup_rows_per_dst=[0, 2, 3132, 10, 816, 0, 56, 0] combine_rows_per_src=[8188, 7990, 8188, 8036, 8190, 8192, 8190, 4546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57244 payload_rows_remote=49052 dup_rows=8292 dup_rows_remote=8280 dedup_ratio=0.126526 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 3477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2526, 0, 0, 0, 0, 3747, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 0, 0, 12, 0, 0, 3946, 1840, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8034, 8192, 8030, 5370, 8192, 7554, 3680] dup_rows_per_dst=[12, 1376, 4752, 1688, 312, 0, 152, 0] combine_rows_per_src=[8192, 8034, 8192, 8030, 5370, 8192, 7554, 3680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=374542336 dispatch_meta_bytes=228088 combine_payload_bytes=374542336 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=374770424 ingress_bytes=814944256 link_bytes=814944256 token_bytes=749312760 weight_movement_bytes=440401920 total_nvlink_bytes=1189714680 weight_to_token_ratio=0.587741 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=28928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.417322096e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=374814720 dispatch_meta_bytes=229144 combine_payload_bytes=374814720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375043864 ingress_bytes=484915200 link_bytes=484915200 token_bytes=749858584 weight_movement_bytes=110100480 total_nvlink_bytes=859959064 weight_to_token_ratio=0.146828 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.433472876e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=374355968 dispatch_meta_bytes=230024 combine_payload_bytes=374355968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=374585992 ingress_bytes=594556928 link_bytes=594556928 token_bytes=748941960 weight_movement_bytes=220200960 total_nvlink_bytes=969142920 weight_to_token_ratio=0.294016 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.034032286e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52362 payload_rows_remote=44170 dup_rows=13174 dup_rows_remote=13172 dedup_ratio=0.201019 router_tokens_per_expert=[4049, 1313, 4092, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 4067, 0, 4096, 0, 125, 0, 0, 2, 9, 0, 0, 0, 0, 0, 0, 625, 0, 0, 4094, 0, 0, 1354, 2, 14, 1, 0, 10, 0, 0, 4094, 0, 719, 1, 0, 1, 0, 0, 4094, 0, 0, 1, 1] payload_rows_per_dst=[2626, 8188, 8136, 7182, 1658, 8188, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 224, 20, 10412, 2514, 2] combine_rows_per_src=[2626, 8188, 8136, 7182, 1658, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=382283776 dispatch_meta_bytes=216864 combine_payload_bytes=382283776 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382500640 ingress_bytes=492384256 link_bytes=492384256 token_bytes=764784416 weight_movement_bytes=110100480 total_nvlink_bytes=874884896 weight_to_token_ratio=0.143963 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.563372045e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=351604736 dispatch_meta_bytes=229328 combine_payload_bytes=351604736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351834064 ingress_bytes=461705216 link_bytes=461705216 token_bytes=703438800 weight_movement_bytes=110100480 total_nvlink_bytes=813539280 weight_to_token_ratio=0.156517 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.029813081e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59838 payload_rows_remote=51728 dup_rows=5698 dup_rows_remote=3266 dedup_ratio=0.086945 router_tokens_per_expert=[4028, 1354, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 4096, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 613, 0, 0, 4096, 0, 0, 1235, 0, 8, 0, 0, 10, 0, 0, 4096, 0, 950, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0] payload_rows_per_dst=[2708, 8192, 8060, 8192, 8192, 8110, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 102, 1226, 2432, 1920, 18] combine_rows_per_src=[2708, 8192, 8060, 8192, 8192, 8110, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59592 payload_rows_remote=51400 dup_rows=5944 dup_rows_remote=5944 dedup_ratio=0.090698 router_tokens_per_expert=[3748, 453, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 4096, 0, 105, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 396, 0, 0, 4096, 0, 0, 2668, 0, 6, 0, 0, 1, 0, 0, 4096, 0, 951, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[7984, 8192, 7918, 8192, 8192, 5336, 5586, 8192] dup_rows_per_dst=[3876, 0, 0, 212, 792, 12, 1052, 0] combine_rows_per_src=[7984, 8192, 7918, 8192, 8192, 5336, 5586, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=316610560 dispatch_meta_bytes=229368 combine_payload_bytes=316610560 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=316839928 ingress_bytes=426711040 link_bytes=426711040 token_bytes=633450488 weight_movement_bytes=110100480 total_nvlink_bytes=743550968 weight_to_token_ratio=0.173811 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.421206805e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61638 payload_rows_remote=53446 dup_rows=3898 dup_rows_remote=2978 dedup_ratio=0.059479 router_tokens_per_expert=[4034, 677, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 0, 4096, 0, 0, 2615, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 335, 0, 1, 0, 0, 0, 4096, 0, 0, 3, 0] payload_rows_per_dst=[6226, 8192, 8154, 8192, 8192, 6298, 8192, 8192] dup_rows_per_dst=[760, 0, 0, 168, 920, 1368, 676, 6] combine_rows_per_src=[6226, 8192, 8154, 8192, 8192, 6298, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60466 payload_rows_remote=52276 dup_rows=5070 dup_rows_remote=3910 dedup_ratio=0.077362 router_tokens_per_expert=[3959, 1694, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 1, 0, 0, 4064, 1, 4095, 0, 169, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 526, 0, 0, 4095, 0, 0, 1287, 0, 3, 0, 1, 3, 0, 0, 4095, 0, 577, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 0] payload_rows_per_dst=[3388, 8190, 8130, 8190, 8190, 7996, 8190, 8192] dup_rows_per_dst=[0, 0, 4, 346, 1052, 2504, 1160, 4] combine_rows_per_src=[3388, 8190, 8130, 8190, 8190, 7996, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61676 payload_rows_remote=53484 dup_rows=3860 dup_rows_remote=3578 dedup_ratio=0.058899 router_tokens_per_expert=[3973, 585, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 4096, 0, 140, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 556, 0, 0, 4096, 0, 0, 2305, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 678, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8006, 8192, 8100, 8192, 8192, 4610, 8192, 8192] dup_rows_per_dst=[1110, 0, 0, 282, 1112, 0, 1356, 0] combine_rows_per_src=[8006, 8192, 8100, 8192, 8192, 4610, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=370786304 dispatch_meta_bytes=219976 combine_payload_bytes=370786304 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=371006280 ingress_bytes=590987264 link_bytes=590987264 token_bytes=741792584 weight_movement_bytes=220200960 total_nvlink_bytes=961993544 weight_to_token_ratio=0.296850 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=15488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.027824053e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=368435200 dispatch_meta_bytes=229376 combine_payload_bytes=368435200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368664576 ingress_bytes=478535680 link_bytes=478535680 token_bytes=737099776 weight_movement_bytes=110100480 total_nvlink_bytes=847200256 weight_to_token_ratio=0.149370 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.322522531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=383100928 dispatch_meta_bytes=225696 combine_payload_bytes=383100928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383326624 ingress_bytes=493201408 link_bytes=493201408 token_bytes=766427552 weight_movement_bytes=110100480 total_nvlink_bytes=876528032 weight_to_token_ratio=0.143654 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=28928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.577583662e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53668 payload_rows_remote=45476 dup_rows=11868 dup_rows_remote=1806 dedup_ratio=0.181091 router_tokens_per_expert=[3717, 1314, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3859, 0, 4095, 0, 75, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 338, 0, 0, 4096, 0, 0, 2451, 0, 95, 0, 0, 59, 0, 0, 4096, 0, 361, 0, 5, 0, 0, 0, 4096, 0, 0, 18, 0] payload_rows_per_dst=[8192, 8176, 8186, 7080, 8192, 4904, 778, 8160] dup_rows_per_dst=[10062, 4, 710, 150, 676, 188, 72, 6] combine_rows_per_src=[8192, 8176, 8186, 7080, 8192, 4904, 778, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60384 payload_rows_remote=52246 dup_rows=5152 dup_rows_remote=5152 dedup_ratio=0.078613 router_tokens_per_expert=[4032, 1261, 4094, 0, 0, 4, 2, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4068, 0, 4094, 0, 315, 0, 0, 2, 3, 2, 0, 0, 0, 0, 0, 369, 0, 0, 4096, 0, 0, 1548, 0, 40, 1, 0, 6, 0, 0, 4093, 0, 632, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0] payload_rows_per_dst=[8128, 8192, 8138, 8192, 8192, 3162, 8192, 8188] dup_rows_per_dst=[2470, 16, 0, 640, 738, 16, 1272, 0] combine_rows_per_src=[8128, 8192, 8138, 8192, 8192, 3162, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=374714368 dispatch_meta_bytes=224744 combine_payload_bytes=374714368 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=374939112 ingress_bytes=594915328 link_bytes=594915328 token_bytes=749653480 weight_movement_bytes=220200960 total_nvlink_bytes=969854440 weight_to_token_ratio=0.293737 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=27008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.034655602e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=383373312 dispatch_meta_bytes=228248 combine_payload_bytes=383373312 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=383601560 ingress_bytes=713674752 link_bytes=713674752 token_bytes=766974872 weight_movement_bytes=330301440 total_nvlink_bytes=1097276312 weight_to_token_ratio=0.430655 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.241197773e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63152 payload_rows_remote=54960 dup_rows=2384 dup_rows_remote=2382 dedup_ratio=0.036377 router_tokens_per_expert=[0, 1, 4094, 0, 0, 0, 0, 3818, 4095, 0, 3263, 0, 0, 1, 0, 0, 0, 0, 4090, 2, 1, 1, 643, 0, 0, 2, 0, 0, 1, 0, 2, 0, 0, 3, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 4090, 1, 0, 4094, 0, 0, 0, 468, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[7638, 6526, 8192, 8190, 8042, 8182, 8190, 8192] dup_rows_per_dst=[0, 2, 1282, 6, 6, 150, 936, 2] combine_rows_per_src=[7638, 6526, 8192, 8190, 8042, 8182, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=325971968 dispatch_meta_bytes=189128 combine_payload_bytes=325971968 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=326161096 ingress_bytes=656273408 link_bytes=656273408 token_bytes=652133064 weight_movement_bytes=330301440 total_nvlink_bytes=982434504 weight_to_token_ratio=0.506494 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.141367395e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=374499328 dispatch_meta_bytes=229592 combine_payload_bytes=374499328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374728920 ingress_bytes=484599808 link_bytes=484599808 token_bytes=749228248 weight_movement_bytes=110100480 total_nvlink_bytes=859328728 weight_to_token_ratio=0.146952 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.427987691e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63228 payload_rows_remote=55040 dup_rows=2308 dup_rows_remote=2308 dedup_ratio=0.035217 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 3614, 4096, 0, 3426, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7228, 6852, 8192, 8192, 8192, 8188, 8192, 8192] dup_rows_per_dst=[0, 0, 1006, 0, 0, 0, 1302, 0] combine_rows_per_src=[7228, 6852, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60732 payload_rows_remote=52760 dup_rows=4804 dup_rows_remote=1048 dedup_ratio=0.073303 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 3169, 4096, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 1, 4096, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6338, 7972, 5464, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 3756, 618, 0, 0, 2, 428, 0] combine_rows_per_src=[6338, 7972, 5464, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=393953280 dispatch_meta_bytes=229368 combine_payload_bytes=393953280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394182648 ingress_bytes=504053760 link_bytes=504053760 token_bytes=788135928 weight_movement_bytes=110100480 total_nvlink_bytes=898236408 weight_to_token_ratio=0.139697 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.766323912e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61694 payload_rows_remote=53502 dup_rows=3842 dup_rows_remote=3838 dedup_ratio=0.058624 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 3313, 4096, 0, 2958, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 2, 4096, 0, 0, 0, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6626, 5916, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2490, 0, 4, 0, 1348, 0] combine_rows_per_src=[6626, 5916, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64298 payload_rows_remote=56106 dup_rows=1238 dup_rows_remote=464 dedup_ratio=0.018890 router_tokens_per_expert=[0, 0, 4095, 0, 1, 0, 0, 3860, 4095, 0, 3718, 1, 0, 0, 0, 0, 0, 0, 4062, 3, 1, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4092, 0, 0, 4095, 0, 0, 0, 385, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7722, 7436, 8192, 8190, 8190, 8184, 8192, 8192] dup_rows_per_dst=[0, 2, 450, 0, 12, 0, 774, 0] combine_rows_per_src=[7722, 7436, 8192, 8190, 8190, 8184, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=394526720 dispatch_meta_bytes=229392 combine_payload_bytes=394526720 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=394756112 ingress_bytes=614727680 link_bytes=614727680 token_bytes=789282832 weight_movement_bytes=220200960 total_nvlink_bytes=1009483792 weight_to_token_ratio=0.278989 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.069112541e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63328 payload_rows_remote=55136 dup_rows=2208 dup_rows_remote=2208 dedup_ratio=0.033691 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 3370, 4096, 0, 3722, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6740, 7444, 8192, 8192, 8192, 8184, 8192, 8192] dup_rows_per_dst=[0, 0, 1366, 0, 0, 0, 842, 0] combine_rows_per_src=[6740, 7444, 8192, 8192, 8192, 8184, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=378183680 dispatch_meta_bytes=215232 combine_payload_bytes=378183680 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=378398912 ingress_bytes=598384640 link_bytes=598384640 token_bytes=756582592 weight_movement_bytes=220200960 total_nvlink_bytes=976783552 weight_to_token_ratio=0.291047 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.040689306e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=383502336 dispatch_meta_bytes=229360 combine_payload_bytes=383502336 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=383731696 ingress_bytes=603703296 link_bytes=603703296 token_bytes=767234032 weight_movement_bytes=220200960 total_nvlink_bytes=987434992 weight_to_token_ratio=0.287006 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.049939324e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52870 payload_rows_remote=44904 dup_rows=12666 dup_rows_remote=5142 dedup_ratio=0.193268 router_tokens_per_expert=[0, 11, 4096, 0, 0, 0, 0, 2771, 4096, 0, 3842, 0, 0, 0, 0, 0, 0, 0, 4089, 28, 0, 0, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4037, 0, 4, 4096, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0] payload_rows_per_dst=[7966, 8120, 2724, 8192, 8188, 8082, 1418, 8180] dup_rows_per_dst=[7524, 4838, 38, 0, 118, 0, 138, 10] combine_rows_per_src=[7966, 8120, 2724, 8192, 8188, 8082, 1418, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63550 payload_rows_remote=55370 dup_rows=1986 dup_rows_remote=1028 dedup_ratio=0.030304 router_tokens_per_expert=[0, 2, 4096, 0, 0, 1, 0, 3874, 4096, 0, 3334, 0, 0, 0, 2, 0, 1, 0, 4061, 0, 0, 0, 507, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4090, 1, 0, 4094, 0, 0, 0, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7750, 6672, 8180, 8190, 8192, 8182, 8192, 8192] dup_rows_per_dst=[4, 0, 958, 0, 16, 0, 1008, 0] combine_rows_per_src=[7750, 6672, 8180, 8190, 8192, 8182, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=402167808 dispatch_meta_bytes=226280 combine_payload_bytes=402167808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402394088 ingress_bytes=512268288 link_bytes=512268288 token_bytes=804561896 weight_movement_bytes=110100480 total_nvlink_bytes=914662376 weight_to_token_ratio=0.136845 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.909188065e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57610 payload_rows_remote=49418 dup_rows=7926 dup_rows_remote=7724 dedup_ratio=0.120941 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4087, 4094, 0, 0, 0, 1, 0, 1, 4, 2, 0, 0, 633, 0, 0, 4084, 0, 26, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4094, 2, 944, 2402, 4094, 4094, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 2, 0, 2, 2, 0, 0, 4094, 0, 0, 101, 0] payload_rows_per_dst=[8188, 8176, 1270, 7212, 8190, 8190, 8192, 8192] dup_rows_per_dst=[0, 2, 8, 48, 7658, 2, 6, 202] combine_rows_per_src=[8188, 8176, 1270, 7212, 8190, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=395214848 dispatch_meta_bytes=229376 combine_payload_bytes=395214848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=395444224 ingress_bytes=505315328 link_bytes=505315328 token_bytes=790659072 weight_movement_bytes=110100480 total_nvlink_bytes=900759552 weight_to_token_ratio=0.139252 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.788264655e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=321871872 dispatch_meta_bytes=200184 combine_payload_bytes=321871872 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=322072056 ingress_bytes=542072832 link_bytes=542072832 token_bytes=643943928 weight_movement_bytes=220200960 total_nvlink_bytes=864144888 weight_to_token_ratio=0.341957 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.427538105e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=396892160 dispatch_meta_bytes=225592 combine_payload_bytes=396892160 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=397117752 ingress_bytes=727193600 link_bytes=727193600 token_bytes=794009912 weight_movement_bytes=330301440 total_nvlink_bytes=1124311352 weight_to_token_ratio=0.415992 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.264709273e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61976 payload_rows_remote=53784 dup_rows=3560 dup_rows_remote=3560 dedup_ratio=0.054321 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1345, 0, 0, 4095, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 847, 1700, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 85, 0] payload_rows_per_dst=[8192, 8192, 7262, 8190, 5564, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2260, 242, 888, 0, 0, 170] combine_rows_per_src=[8192, 8192, 7262, 8190, 5564, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63436 payload_rows_remote=55244 dup_rows=2100 dup_rows_remote=2100 dedup_ratio=0.032043 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 685, 0, 0, 4096, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1054, 1992, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 216, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6092, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 1372, 296, 0, 0, 0, 432] combine_rows_per_src=[8192, 8192, 8192, 8192, 6092, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=354228224 dispatch_meta_bytes=228568 combine_payload_bytes=354228224 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=354456792 ingress_bytes=684529664 link_bytes=684529664 token_bytes=708685016 weight_movement_bytes=330301440 total_nvlink_bytes=1038986456 weight_to_token_ratio=0.466077 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.190509671e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63310 payload_rows_remote=57342 dup_rows=2226 dup_rows_remote=2226 dedup_ratio=0.033966 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 955, 0, 0, 4090, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1176, 1808, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 138, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5968, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 1898, 48, 0, 0, 4, 276] combine_rows_per_src=[8192, 8192, 8192, 8192, 5968, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59090 payload_rows_remote=50900 dup_rows=6446 dup_rows_remote=6444 dedup_ratio=0.098358 router_tokens_per_expert=[1, 0, 0, 0, 0, 1, 0, 1, 4096, 4095, 0, 0, 1, 0, 0, 0, 2, 0, 1, 0, 880, 0, 0, 4091, 0, 60, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4089, 0, 1152, 1892, 4094, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 118, 0] payload_rows_per_dst=[8192, 8192, 1766, 8180, 8190, 8190, 8190, 8190] dup_rows_per_dst=[4, 2, 0, 120, 6082, 0, 2, 236] combine_rows_per_src=[8192, 8192, 1766, 8180, 8190, 8190, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=385523712 dispatch_meta_bytes=229376 combine_payload_bytes=385523712 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=385753088 ingress_bytes=605724672 link_bytes=605724672 token_bytes=771276800 weight_movement_bytes=220200960 total_nvlink_bytes=991477760 weight_to_token_ratio=0.285502 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.053454829e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=395988992 dispatch_meta_bytes=229376 combine_payload_bytes=395988992 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=396218368 ingress_bytes=616189952 link_bytes=616189952 token_bytes=792207360 weight_movement_bytes=220200960 total_nvlink_bytes=1012408320 weight_to_token_ratio=0.277959 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.071655672e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63892 payload_rows_remote=55702 dup_rows=1644 dup_rows_remote=1590 dedup_ratio=0.025085 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 4093, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1750, 1527, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 243, 0] payload_rows_per_dst=[8192, 8192, 8190, 8190, 6552, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 1102, 54, 2, 0, 0, 486] combine_rows_per_src=[8192, 8192, 8190, 8190, 6552, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=411027456 dispatch_meta_bytes=238272 combine_payload_bytes=411027456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=411265728 ingress_bytes=631228416 link_bytes=631228416 token_bytes=822293184 weight_movement_bytes=220200960 total_nvlink_bytes=1042494144 weight_to_token_ratio=0.267789 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=14592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097810035e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59248 payload_rows_remote=51056 dup_rows=6288 dup_rows_remote=6288 dedup_ratio=0.095947 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1418, 0, 0, 4094, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 1055, 1118, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 445, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 4348, 8178, 8174, 5780] dup_rows_per_dst=[0, 12, 5794, 120, 0, 24, 0, 338] combine_rows_per_src=[8192, 8192, 8192, 8192, 4348, 8178, 8174, 5780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63328 payload_rows_remote=55136 dup_rows=2208 dup_rows_remote=362 dedup_ratio=0.033691 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 922, 0, 0, 4092, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 930, 2069, 4091, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 141, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 6000, 8186, 8192, 8184] dup_rows_per_dst=[0, 2, 1846, 68, 2, 0, 8, 282] combine_rows_per_src=[8192, 8190, 8192, 8192, 6000, 8186, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=364851200 dispatch_meta_bytes=229376 combine_payload_bytes=364851200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=365080576 ingress_bytes=474951680 link_bytes=474951680 token_bytes=729931776 weight_movement_bytes=110100480 total_nvlink_bytes=840032256 weight_to_token_ratio=0.150837 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.260190876e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58070 payload_rows_remote=49878 dup_rows=7466 dup_rows_remote=7430 dedup_ratio=0.113922 router_tokens_per_expert=[0, 0, 4096, 0, 0, 1, 1, 4095, 0, 2, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 233, 3982, 0, 207, 0, 0, 567, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 4096, 0, 3192, 0, 0, 3, 0, 2, 0, 2, 1, 0] payload_rows_per_dst=[8192, 8192, 8186, 8188, 876, 8190, 8054, 8192] dup_rows_per_dst=[2, 6, 6, 0, 6, 1136, 6274, 36] combine_rows_per_src=[8192, 8192, 8186, 8188, 876, 8190, 8054, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=399271936 dispatch_meta_bytes=229168 combine_payload_bytes=399271936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399501104 ingress_bytes=509372416 link_bytes=509372416 token_bytes=798773040 weight_movement_bytes=110100480 total_nvlink_bytes=908873520 weight_to_token_ratio=0.137837 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.858824088e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=365969408 dispatch_meta_bytes=229376 combine_payload_bytes=365969408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366198784 ingress_bytes=476069888 link_bytes=476069888 token_bytes=732168192 weight_movement_bytes=110100480 total_nvlink_bytes=842268672 weight_to_token_ratio=0.150376 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.279638353e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59492 payload_rows_remote=51300 dup_rows=6044 dup_rows_remote=5716 dedup_ratio=0.092224 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1022, 3976, 0, 174, 0, 0, 164, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 2492, 8192, 7852, 8188] dup_rows_per_dst=[0, 0, 0, 0, 344, 328, 5372, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 2492, 8192, 7852, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=395214848 dispatch_meta_bytes=221992 combine_payload_bytes=395214848 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=395436840 ingress_bytes=615415808 link_bytes=615415808 token_bytes=790651688 weight_movement_bytes=220200960 total_nvlink_bytes=1010852648 weight_to_token_ratio=0.278506 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=23936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.070309309e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61896 payload_rows_remote=53704 dup_rows=3640 dup_rows_remote=3640 dedup_ratio=0.055542 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1675, 3718, 0, 181, 0, 0, 334, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 2284, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8184, 8192, 4568, 8184] dup_rows_per_dst=[0, 0, 8, 0, 2964, 668, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8184, 8192, 4568, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62128 payload_rows_remote=53942 dup_rows=3408 dup_rows_remote=684 dedup_ratio=0.052002 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 911, 3914, 0, 630, 0, 0, 342, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 2398, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8186, 8192, 4796, 8186] dup_rows_per_dst=[0, 0, 0, 0, 2724, 684, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8186, 8192, 4796, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=357525504 dispatch_meta_bytes=229232 combine_payload_bytes=357525504 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=357754736 ingress_bytes=577726464 link_bytes=577726464 token_bytes=715280240 weight_movement_bytes=220200960 total_nvlink_bytes=935481200 weight_to_token_ratio=0.307853 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.004761341e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=367718400 dispatch_meta_bytes=228064 combine_payload_bytes=367718400 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=367946464 ingress_bytes=587919360 link_bytes=587919360 token_bytes=735664864 weight_movement_bytes=220200960 total_nvlink_bytes=955865824 weight_to_token_ratio=0.299322 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.022488463e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60586 payload_rows_remote=52518 dup_rows=4950 dup_rows_remote=426 dedup_ratio=0.075531 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 1468, 4001, 1, 250, 0, 0, 162, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 1, 2294, 0, 0, 11, 1, 0, 0, 1, 0, 0] payload_rows_per_dst=[8190, 8190, 8192, 8192, 3372, 8190, 8068, 8192] dup_rows_per_dst=[0, 0, 10, 2, 66, 324, 4524, 24] combine_rows_per_src=[8190, 8190, 8192, 8192, 3372, 8190, 8068, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=384950272 dispatch_meta_bytes=229376 combine_payload_bytes=384950272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385179648 ingress_bytes=495050752 link_bytes=495050752 token_bytes=770129920 weight_movement_bytes=110100480 total_nvlink_bytes=880230400 weight_to_token_ratio=0.142964 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.609746796e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=386656256 dispatch_meta_bytes=218504 combine_payload_bytes=386656256 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=386874760 ingress_bytes=827058176 link_bytes=827058176 token_bytes=773531016 weight_movement_bytes=440401920 total_nvlink_bytes=1213932936 weight_to_token_ratio=0.569340 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.438390196e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6966873848 prefetch_bytes=1211105280 reduce_bytes=880803840 weight_movement_bytes=2091909120 egress_bytes=3484559096 ingress_bytes=5574223872 link_bytes=5574223872 total_nvlink_bytes=9058782968 weight_to_token_ratio=0.300265 prefetch_share_of_ingress=0.217269 dedup_ratio=0.138354 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=8 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=9906 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.694492116e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7401996608 prefetch_bytes=1101004800 reduce_bytes=1101004800 weight_movement_bytes=2202009600 egress_bytes=3702133056 ingress_bytes=5901873152 link_bytes=5901873152 total_nvlink_bytes=9604006208 weight_to_token_ratio=0.297489 prefetch_share_of_ingress=0.186552 dedup_ratio=0.093112 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=10 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=9525 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.026432810e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58304 payload_rows_remote=50148 dup_rows=7232 dup_rows_remote=7194 dedup_ratio=0.110352 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1286, 3805, 0, 169, 0, 0, 465, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4031, 0, 2509, 0, 0, 41, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8156, 8184, 8162, 8192, 8178, 4286, 5018, 8128] dup_rows_per_dst=[38, 0, 0, 4, 6736, 438, 0, 16] combine_rows_per_src=[8156, 8184, 8162, 8192, 8178, 4286, 5018, 8128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62024 payload_rows_remote=53832 dup_rows=3512 dup_rows_remote=3512 dedup_ratio=0.053589 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1927, 3694, 0, 102, 0, 0, 102, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2363, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8146, 8192, 4726, 8192] dup_rows_per_dst=[0, 0, 8, 0, 3300, 204, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8146, 8192, 4726, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7298579128 prefetch_bytes=1211105280 reduce_bytes=550502400 weight_movement_bytes=1761607680 egress_bytes=3650411192 ingress_bytes=5409775616 link_bytes=5409775616 total_nvlink_bytes=9060186808 weight_to_token_ratio=0.241363 prefetch_share_of_ingress=0.223873 dedup_ratio=0.102295 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=5 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=9525 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.408489551e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63604 payload_rows_remote=55412 dup_rows=1932 dup_rows_remote=1928 dedup_ratio=0.029480 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1049, 3760, 0, 172, 0, 0, 63, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 3142, 0, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8190, 8172, 8192, 6284, 8192] dup_rows_per_dst=[0, 4, 4, 0, 1790, 128, 0, 6] combine_rows_per_src=[8190, 8192, 8192, 8190, 8172, 8192, 6284, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=376449024 dispatch_meta_bytes=211776 combine_payload_bytes=376449024 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=376660800 ingress_bytes=706750464 link_bytes=706750464 token_bytes=753109824 weight_movement_bytes=330301440 total_nvlink_bytes=1083411264 weight_to_token_ratio=0.438583 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=21120 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.229155297e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7536739104 prefetch_bytes=880803840 reduce_bytes=1651507200 weight_movement_bytes=2532311040 egress_bytes=3769510688 ingress_bytes=6299539456 link_bytes=6299539456 total_nvlink_bytes=10069050144 weight_to_token_ratio=0.335996 prefetch_share_of_ingress=0.139820 dedup_ratio=0.081516 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=15 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044118 gemm_inner_pad_upper_bound=8890 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.095593521e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=359460864 dispatch_meta_bytes=229368 combine_payload_bytes=359460864 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=359690232 ingress_bytes=579661824 link_bytes=579661824 token_bytes=719151096 weight_movement_bytes=220200960 total_nvlink_bytes=939352056 weight_to_token_ratio=0.306196 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.008127250e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7346785992 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3674533576 ingress_bytes=5654061056 link_bytes=5654061056 total_nvlink_bytes=9328594632 weight_to_token_ratio=0.269752 prefetch_share_of_ingress=0.194728 dedup_ratio=0.103622 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=9017 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.833342110e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=385867776 dispatch_meta_bytes=229376 combine_payload_bytes=385867776 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=386097152 ingress_bytes=606068736 link_bytes=606068736 token_bytes=771964928 weight_movement_bytes=220200960 total_nvlink_bytes=992165888 weight_to_token_ratio=0.285247 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.054053213e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=397193216 dispatch_meta_bytes=229360 combine_payload_bytes=397193216 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=397422576 ingress_bytes=617394176 link_bytes=617394176 token_bytes=794615792 weight_movement_bytes=220200960 total_nvlink_bytes=1014816752 weight_to_token_ratio=0.277116 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.073750016e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6673375560 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=3337789768 ingress_bytes=5317394432 link_bytes=5317394432 total_nvlink_bytes=8655184200 weight_to_token_ratio=0.296972 prefetch_share_of_ingress=0.186351 dedup_ratio=0.170300 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=9525 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.247823478e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7554303864 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=3778287480 ingress_bytes=5757825024 link_bytes=5757825024 total_nvlink_bytes=9536112504 weight_to_token_ratio=0.262342 prefetch_share_of_ingress=0.172097 dedup_ratio=0.077454 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=9525 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.001380472e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7502689152 prefetch_bytes=990904320 reduce_bytes=1431306240 weight_movement_bytes=2422210560 egress_bytes=3752491904 ingress_bytes=6172407808 link_bytes=6172407808 total_nvlink_bytes=9924899712 weight_to_token_ratio=0.322846 prefetch_share_of_ingress=0.160538 dedup_ratio=0.085495 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=13 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043750 gemm_inner_pad_upper_bound=8763 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.073483236e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none + [2026-08-26 18:09:52] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 365.9 | throughput per GPU (TFLOP/s/GPU): 337.5 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.046635E+01 | loss scale: 1.0 | grad norm: 8.623 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44788 payload_rows_remote=37758 dup_rows=20748 dup_rows_remote=15810 dedup_ratio=0.316589 router_tokens_per_expert=[400, 156, 534, 693, 8, 27, 334, 319, 722, 607, 329, 1304, 0, 388, 4, 839, 582, 392, 201, 1, 921, 1885, 7, 488, 269, 802, 773, 507, 45, 487, 610, 0, 35, 15, 207, 323, 28, 0, 117, 1485, 845, 600, 1382, 120, 535, 262, 114, 866, 16, 247, 561, 1410, 1949, 235, 3, 417, 212, 388, 752, 1712, 1562, 1448, 287, 1] payload_rows_per_dst=[4214, 4430, 4168, 6242, 5416, 6878, 6410, 7030] dup_rows_per_dst=[1124, 1348, 1016, 3352, 2128, 3576, 3266, 4938] combine_rows_per_src=[4214, 4430, 4168, 6242, 5416, 6878, 6410, 7030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45636 payload_rows_remote=39200 dup_rows=19900 dup_rows_remote=16934 dedup_ratio=0.303650 router_tokens_per_expert=[348, 248, 886, 628, 21, 91, 114, 337, 1012, 736, 451, 1438, 11, 149, 6, 952, 660, 488, 109, 3, 1046, 1460, 10, 447, 259, 590, 1060, 653, 14, 346, 353, 1, 387, 109, 229, 208, 14, 0, 22, 733, 1096, 612, 856, 79, 192, 304, 249, 709, 48, 85, 439, 1347, 1188, 85, 43, 596, 257, 509, 1439, 2227, 2369, 982, 428, 0] payload_rows_per_dst=[6220, 5426, 4518, 5792, 5936, 6436, 5526, 5782] dup_rows_per_dst=[3580, 2480, 1008, 2364, 2206, 2966, 2136, 3160] combine_rows_per_src=[6220, 5426, 4518, 5792, 5936, 6436, 5526, 5782] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46008 payload_rows_remote=40420 dup_rows=19528 dup_rows_remote=16964 dedup_ratio=0.297974 router_tokens_per_expert=[590, 86, 331, 719, 15, 24, 172, 132, 575, 847, 867, 1782, 0, 215, 3, 717, 963, 444, 233, 2, 924, 1586, 18, 696, 304, 630, 1072, 510, 8, 435, 418, 0, 42, 6, 265, 143, 15, 0, 59, 801, 717, 965, 1196, 101, 68, 401, 306, 1112, 2, 232, 268, 826, 1530, 418, 4, 796, 275, 359, 1014, 1649, 2367, 1295, 218, 0] payload_rows_per_dst=[5472, 4854, 5064, 6746, 5782, 6814, 5588, 5688] dup_rows_per_dst=[1964, 1594, 1496, 3572, 1614, 3698, 2564, 3026] combine_rows_per_src=[5472, 4854, 5064, 6746, 5782, 6814, 5588, 5688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45926 payload_rows_remote=40008 dup_rows=19610 dup_rows_remote=17776 dedup_ratio=0.299225 router_tokens_per_expert=[385, 101, 323, 619, 46, 34, 421, 147, 559, 492, 969, 1580, 0, 390, 0, 711, 1333, 203, 361, 0, 938, 2020, 8, 531, 255, 1159, 925, 632, 16, 532, 267, 2, 38, 14, 370, 204, 17, 0, 104, 715, 680, 618, 1549, 46, 87, 469, 111, 929, 14, 116, 139, 715, 2034, 540, 26, 509, 280, 352, 936, 2285, 2414, 344, 154, 0] payload_rows_per_dst=[6182, 6086, 6806, 5462, 5918, 4498, 5790, 5184] dup_rows_per_dst=[2540, 3316, 3874, 2114, 1834, 1490, 2396, 2046] combine_rows_per_src=[6182, 6086, 6806, 5462, 5918, 4498, 5790, 5184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45382 payload_rows_remote=40026 dup_rows=20154 dup_rows_remote=18430 dedup_ratio=0.307526 router_tokens_per_expert=[566, 152, 948, 900, 14, 17, 158, 274, 917, 470, 444, 1018, 0, 384, 0, 786, 1131, 234, 327, 1, 788, 1649, 19, 253, 389, 580, 855, 592, 4, 678, 438, 4, 73, 22, 362, 138, 28, 1, 89, 1093, 923, 972, 857, 148, 168, 719, 156, 702, 5, 212, 235, 1157, 2354, 72, 14, 669, 299, 252, 1047, 1745, 1731, 1395, 140, 0] payload_rows_per_dst=[6078, 5392, 6192, 5356, 5172, 5314, 6498, 5380] dup_rows_per_dst=[3470, 2646, 2612, 1724, 1902, 2262, 2938, 2600] combine_rows_per_src=[6078, 5392, 6192, 5356, 5172, 5314, 6498, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45256 payload_rows_remote=39532 dup_rows=20280 dup_rows_remote=17430 dedup_ratio=0.309448 router_tokens_per_expert=[581, 105, 305, 461, 43, 46, 229, 199, 660, 830, 208, 1318, 0, 383, 0, 888, 655, 339, 421, 0, 577, 1829, 13, 665, 625, 239, 1009, 332, 18, 1258, 646, 0, 22, 32, 145, 183, 16, 2, 100, 1550, 1071, 557, 871, 105, 368, 394, 135, 923, 20, 314, 305, 1338, 2029, 414, 6, 455, 229, 302, 1017, 1876, 1393, 1439, 258, 17] payload_rows_per_dst=[5736, 5724, 6572, 5860, 6290, 4896, 5940, 4238] dup_rows_per_dst=[1954, 2850, 3670, 2394, 3254, 2210, 2578, 1370] combine_rows_per_src=[5736, 5724, 6572, 5860, 6290, 4896, 5940, 4238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=270649344 dispatch_meta_bytes=214272 combine_payload_bytes=270649344 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=270863616 ingress_bytes=711051264 link_bytes=711051264 token_bytes=541512960 weight_movement_bytes=440401920 total_nvlink_bytes=981914880 weight_to_token_ratio=0.813280 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=9216 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.236635096e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38538 dup_rows=21160 dup_rows_remote=18544 dedup_ratio=0.322876 router_tokens_per_expert=[569, 101, 488, 717, 11, 41, 269, 281, 798, 668, 440, 1243, 0, 268, 0, 940, 1087, 348, 386, 12, 1034, 1104, 24, 260, 399, 694, 1036, 732, 47, 497, 497, 8, 154, 12, 274, 172, 51, 1, 95, 1123, 700, 828, 1052, 141, 319, 666, 71, 1053, 27, 143, 526, 782, 1567, 528, 8, 795, 186, 474, 1056, 1750, 1777, 1312, 126, 0] payload_rows_per_dst=[5838, 6342, 6290, 5678, 6408, 5438, 4076, 4306] dup_rows_per_dst=[2616, 4812, 2914, 2142, 3534, 2426, 1542, 1174] combine_rows_per_src=[5838, 6342, 6290, 5678, 6408, 5438, 4076, 4306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45606 payload_rows_remote=39130 dup_rows=19930 dup_rows_remote=17070 dedup_ratio=0.304108 router_tokens_per_expert=[352, 147, 444, 545, 11, 45, 413, 265, 1115, 790, 358, 721, 1, 357, 10, 766, 789, 339, 399, 12, 1001, 1758, 18, 352, 390, 722, 871, 433, 35, 502, 416, 0, 74, 25, 537, 189, 17, 0, 179, 876, 964, 704, 1567, 92, 106, 362, 103, 986, 25, 239, 245, 1210, 1226, 427, 81, 691, 486, 301, 859, 2052, 2174, 1231, 357, 6] payload_rows_per_dst=[5858, 5480, 6476, 5182, 5818, 4980, 5794, 6018] dup_rows_per_dst=[2690, 2756, 2860, 1556, 2324, 1654, 2494, 3596] combine_rows_per_src=[5858, 5480, 6476, 5182, 5818, 4980, 5794, 6018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=289730560 dispatch_meta_bytes=229536 combine_payload_bytes=289730560 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=289960096 ingress_bytes=509931520 link_bytes=509931520 token_bytes=579690656 weight_movement_bytes=220200960 total_nvlink_bytes=799891616 weight_to_token_ratio=0.379859 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=11776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.868547826e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280985600 dispatch_meta_bytes=224536 combine_payload_bytes=280985600 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=281210136 ingress_bytes=501186560 link_bytes=501186560 token_bytes=562195736 weight_movement_bytes=220200960 total_nvlink_bytes=782396696 weight_to_token_ratio=0.391680 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=7296 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.716458589e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286777344 dispatch_meta_bytes=231136 combine_payload_bytes=286777344 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=287008480 ingress_bytes=506978304 link_bytes=506978304 token_bytes=573785824 weight_movement_bytes=220200960 total_nvlink_bytes=793986784 weight_to_token_ratio=0.383769 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.817186543e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286906368 dispatch_meta_bytes=233824 combine_payload_bytes=286906368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=287140192 ingress_bytes=397006848 link_bytes=397006848 token_bytes=574046560 weight_movement_bytes=110100480 total_nvlink_bytes=684147040 weight_to_token_ratio=0.191797 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.904602051e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=283365376 dispatch_meta_bytes=227848 combine_payload_bytes=283365376 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=283593224 ingress_bytes=503566336 link_bytes=503566336 token_bytes=566958600 weight_movement_bytes=220200960 total_nvlink_bytes=787159560 weight_to_token_ratio=0.388390 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.757846807e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276240384 dispatch_meta_bytes=228328 combine_payload_bytes=276240384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276468712 ingress_bytes=386340864 link_bytes=386340864 token_bytes=552709096 weight_movement_bytes=110100480 total_nvlink_bytes=662809576 weight_to_token_ratio=0.199201 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.719103046e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280483840 dispatch_meta_bytes=224800 combine_payload_bytes=280483840 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280708640 ingress_bytes=500684800 link_bytes=500684800 token_bytes=561192480 weight_movement_bytes=220200960 total_nvlink_bytes=781393440 weight_to_token_ratio=0.392380 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.707732157e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54252 payload_rows_remote=46064 dup_rows=11284 dup_rows_remote=3084 dedup_ratio=0.172180 router_tokens_per_expert=[0, 1, 0, 644, 0, 0, 0, 40, 0, 0, 2, 1, 2448, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 1, 6, 4084, 0, 0, 0, 0, 5, 2, 2, 0, 3552, 1, 0, 0, 4, 0, 37, 4065, 0, 5, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 1498, 2, 0, 2, 0, 1986, 4091, 0, 2093, 0, 4, 0] payload_rows_per_dst=[1368, 4900, 8152, 8178, 7138, 8138, 8190, 8188] dup_rows_per_dst=[2, 2, 14, 8, 50, 6, 3002, 8200] combine_rows_per_src=[1368, 4900, 8152, 8178, 7138, 8138, 8190, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57680 payload_rows_remote=49488 dup_rows=7856 dup_rows_remote=5240 dedup_ratio=0.119873 router_tokens_per_expert=[0, 0, 0, 542, 2, 0, 0, 13, 2, 0, 1, 0, 2810, 0, 0, 0, 0, 0, 0, 4093, 3, 3, 0, 0, 4090, 0, 0, 0, 0, 1, 0, 0, 0, 3265, 0, 0, 0, 3, 0, 1, 4061, 0, 0, 1, 4096, 0, 0, 0, 2, 1, 0, 0, 1297, 6, 0, 2, 0, 2427, 4095, 0, 1951, 0, 0, 0] payload_rows_per_dst=[5392, 5626, 8190, 8180, 6538, 8124, 8192, 7438] dup_rows_per_dst=[602, 0, 8, 2, 0, 0, 2616, 4628] combine_rows_per_src=[5392, 5626, 8190, 8180, 6538, 8124, 8192, 7438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60570 payload_rows_remote=52584 dup_rows=4966 dup_rows_remote=4960 dedup_ratio=0.075775 router_tokens_per_expert=[0, 0, 0, 821, 2, 0, 0, 70, 0, 0, 27, 0, 3024, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 18, 2, 0, 3, 3824, 1, 0, 0, 0, 0, 61, 3992, 0, 4, 0, 4093, 0, 0, 0, 0, 1, 0, 2, 1228, 0, 0, 0, 0, 1638, 4092, 0, 1679, 0, 2, 0] payload_rows_per_dst=[8192, 6068, 8190, 8190, 7676, 7986, 8192, 6076] dup_rows_per_dst=[1778, 34, 2, 26, 102, 6, 2456, 562] combine_rows_per_src=[8192, 6068, 8190, 8190, 7676, 7986, 8192, 6076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60012 payload_rows_remote=53808 dup_rows=5524 dup_rows_remote=5520 dedup_ratio=0.084290 router_tokens_per_expert=[0, 0, 0, 698, 0, 0, 0, 152, 0, 0, 0, 0, 2940, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3096, 0, 0, 0, 0, 0, 8, 4089, 0, 0, 0, 4095, 0, 3, 0, 0, 0, 0, 0, 1007, 0, 0, 0, 0, 2597, 4096, 0, 1799, 0, 0, 0] payload_rows_per_dst=[8192, 5880, 8190, 8186, 6204, 8178, 8190, 6992] dup_rows_per_dst=[1700, 0, 0, 0, 4, 6, 2014, 1800] combine_rows_per_src=[8192, 5880, 8190, 8186, 6204, 8178, 8190, 6992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59152 payload_rows_remote=50994 dup_rows=6384 dup_rows_remote=6382 dedup_ratio=0.097412 router_tokens_per_expert=[0, 0, 0, 371, 0, 1, 4, 116, 1, 0, 2, 0, 2714, 1, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 4079, 0, 0, 0, 0, 1, 0, 0, 0, 2785, 0, 0, 0, 0, 0, 43, 4060, 2, 0, 0, 4095, 0, 0, 0, 0, 2, 1, 0, 1470, 0, 0, 0, 0, 3026, 4094, 0, 1802, 0, 0, 1] payload_rows_per_dst=[8192, 5430, 8192, 8158, 5626, 8124, 8192, 7238] dup_rows_per_dst=[980, 6, 2, 2, 30, 0, 2944, 2420] combine_rows_per_src=[8192, 5430, 8192, 8158, 5626, 8124, 8192, 7238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=330186752 dispatch_meta_bytes=196592 combine_payload_bytes=330186752 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=330383344 ingress_bytes=770588672 link_bytes=770588672 token_bytes=660570096 weight_movement_bytes=440401920 total_nvlink_bytes=1100972016 weight_to_token_ratio=0.666700 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=14336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.340180441e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55272 payload_rows_remote=47096 dup_rows=10264 dup_rows_remote=4368 dedup_ratio=0.156616 router_tokens_per_expert=[0, 0, 0, 564, 0, 0, 0, 25, 0, 0, 5, 0, 2976, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 4, 0, 0, 10, 0, 0, 0, 3073, 0, 0, 0, 0, 0, 25, 4055, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1561, 0, 0, 0, 0, 2323, 4081, 0, 1784, 0, 0, 0] payload_rows_per_dst=[8182, 8176, 8192, 8176, 7664, 8192, 3122, 3568] dup_rows_per_dst=[1158, 5896, 0, 28, 3178, 4, 0, 0] combine_rows_per_src=[8182, 8176, 8192, 8176, 7664, 8192, 3122, 3568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54318 payload_rows_remote=46132 dup_rows=11218 dup_rows_remote=9046 dedup_ratio=0.171173 router_tokens_per_expert=[0, 0, 0, 830, 0, 0, 0, 261, 0, 0, 0, 0, 2618, 0, 0, 6, 0, 2, 1, 4095, 1, 0, 1, 0, 4082, 4, 0, 0, 1, 10, 0, 2, 0, 3081, 0, 0, 1, 0, 0, 36, 4043, 0, 2, 1, 4096, 2, 3, 0, 0, 0, 0, 2, 1633, 3, 0, 0, 6, 2560, 4088, 0, 1296, 1, 0, 0] payload_rows_per_dst=[8186, 8160, 8192, 8168, 7642, 8192, 3274, 2504] dup_rows_per_dst=[2172, 5174, 16, 22, 3714, 116, 2, 2] combine_rows_per_src=[8186, 8160, 8192, 8168, 7642, 8192, 3274, 2504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=354729984 dispatch_meta_bytes=218912 combine_payload_bytes=354729984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=354948896 ingress_bytes=464830464 link_bytes=464830464 token_bytes=709678880 weight_movement_bytes=110100480 total_nvlink_bytes=819779360 weight_to_token_ratio=0.155141 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.084166284e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=376922112 dispatch_meta_bytes=230176 combine_payload_bytes=376922112 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=377152288 ingress_bytes=707223552 link_bytes=707223552 token_bytes=754074400 weight_movement_bytes=330301440 total_nvlink_bytes=1084375840 weight_to_token_ratio=0.438022 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=21760 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.229978075e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=385695744 dispatch_meta_bytes=237312 combine_payload_bytes=385695744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385933056 ingress_bytes=495796224 link_bytes=495796224 token_bytes=771628800 weight_movement_bytes=110100480 total_nvlink_bytes=881729280 weight_to_token_ratio=0.142686 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.622711780e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=365524992 dispatch_meta_bytes=229504 combine_payload_bytes=365524992 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=365754496 ingress_bytes=475625472 link_bytes=475625472 token_bytes=731279488 weight_movement_bytes=110100480 total_nvlink_bytes=841379968 weight_to_token_ratio=0.150559 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.271909228e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56912 payload_rows_remote=48720 dup_rows=8624 dup_rows_remote=8620 dedup_ratio=0.131592 router_tokens_per_expert=[0, 2, 1, 578, 0, 0, 3, 45, 0, 0, 32, 0, 2451, 0, 0, 0, 3, 0, 0, 4094, 0, 1, 0, 0, 4074, 0, 0, 0, 0, 32, 0, 0, 0, 3219, 0, 0, 0, 0, 0, 5, 4063, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 1, 0, 1251, 7, 0, 0, 2, 2835, 4091, 0, 1880, 1, 3, 0] payload_rows_per_dst=[8190, 7214, 8192, 8170, 7424, 8182, 4280, 5260] dup_rows_per_dst=[1250, 2970, 4, 42, 2546, 282, 870, 660] combine_rows_per_src=[8190, 7214, 8192, 8170, 7424, 8182, 4280, 5260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=337584128 dispatch_meta_bytes=205856 combine_payload_bytes=337584128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=337789984 ingress_bytes=447684608 link_bytes=447684608 token_bytes=675374112 weight_movement_bytes=110100480 total_nvlink_bytes=785474592 weight_to_token_ratio=0.163021 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=22016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.785971648e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=330674176 dispatch_meta_bytes=220712 combine_payload_bytes=330674176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=330894888 ingress_bytes=440774656 link_bytes=440774656 token_bytes=661569064 weight_movement_bytes=110100480 total_nvlink_bytes=771669544 weight_to_token_ratio=0.166423 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=27008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.665796218e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56768 payload_rows_remote=49034 dup_rows=8768 dup_rows_remote=5138 dedup_ratio=0.133789 router_tokens_per_expert=[0, 0, 0, 156, 4092, 6, 0, 0, 4080, 3, 0, 0, 0, 0, 1864, 0, 0, 4091, 0, 0, 0, 2, 0, 0, 0, 0, 6, 36, 0, 0, 4090, 0, 2, 2414, 0, 0, 0, 2, 0, 1, 1, 4090, 4, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 1, 2, 3724, 0, 0, 4] payload_rows_per_dst=[4398, 3734, 8184, 8156, 8184, 8190, 8188, 7734] dup_rows_per_dst=[208, 0, 2, 72, 4814, 10, 32, 3630] combine_rows_per_src=[4398, 3734, 8184, 8156, 8184, 8190, 8188, 7734] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=349224960 dispatch_meta_bytes=229360 combine_payload_bytes=349224960 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=349454320 ingress_bytes=569425920 link_bytes=569425920 token_bytes=698679280 weight_movement_bytes=220200960 total_nvlink_bytes=918880240 weight_to_token_ratio=0.315167 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.903253294e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=351475712 dispatch_meta_bytes=216688 combine_payload_bytes=351475712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351692400 ingress_bytes=461576192 link_bytes=461576192 token_bytes=703168112 weight_movement_bytes=110100480 total_nvlink_bytes=813268592 weight_to_token_ratio=0.156578 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.027569141e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60912 payload_rows_remote=52720 dup_rows=4624 dup_rows_remote=4622 dedup_ratio=0.070557 router_tokens_per_expert=[0, 0, 0, 252, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 2055, 0, 2, 4095, 0, 0, 0, 0, 0, 1, 2, 0, 0, 34, 0, 3, 4095, 0, 0, 2014, 0, 0, 0, 1, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 6, 0, 3833, 0, 0, 0] payload_rows_per_dst=[8192, 4110, 8190, 8192, 8182, 8192, 8186, 7668] dup_rows_per_dst=[504, 0, 6, 76, 4026, 2, 0, 10] combine_rows_per_src=[8192, 4110, 8190, 8192, 8182, 8192, 8186, 7668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62524 payload_rows_remote=54336 dup_rows=3012 dup_rows_remote=3012 dedup_ratio=0.045959 router_tokens_per_expert=[0, 0, 0, 170, 4092, 2, 0, 0, 4093, 5, 0, 2, 0, 0, 2662, 0, 0, 4095, 1, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 5, 4095, 1, 0, 1316, 0, 0, 0, 4, 4, 2, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4027, 0, 0, 0] payload_rows_per_dst=[8188, 5336, 8190, 8192, 8192, 8184, 8188, 8054] dup_rows_per_dst=[340, 2, 4, 20, 2646, 0, 0, 0] combine_rows_per_src=[8188, 5336, 8190, 8192, 8192, 8184, 8188, 8054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61848 payload_rows_remote=53660 dup_rows=3688 dup_rows_remote=316 dedup_ratio=0.056274 router_tokens_per_expert=[0, 0, 0, 150, 4096, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 2427, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 1686, 0, 0, 0, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 3920, 0, 0, 1] payload_rows_per_dst=[8192, 4856, 8192, 8192, 8188, 8190, 8192, 7846] dup_rows_per_dst=[300, 0, 0, 12, 3372, 0, 0, 4] combine_rows_per_src=[8192, 4856, 8192, 8192, 8188, 8190, 8192, 7846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60934 payload_rows_remote=52742 dup_rows=4602 dup_rows_remote=4552 dedup_ratio=0.070221 router_tokens_per_expert=[0, 0, 0, 187, 4095, 2, 0, 1, 4095, 1, 0, 0, 0, 0, 2477, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 4096, 0, 0, 1509, 0, 0, 0, 1, 0, 1, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 3988, 0, 0, 8] payload_rows_per_dst=[8192, 6648, 8178, 8192, 5348, 8192, 8190, 7994] dup_rows_per_dst=[378, 2722, 0, 50, 1450, 2, 0, 0] combine_rows_per_src=[8192, 6648, 8178, 8192, 5348, 8192, 8190, 7994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59916 payload_rows_remote=51730 dup_rows=5620 dup_rows_remote=360 dedup_ratio=0.085754 router_tokens_per_expert=[0, 0, 0, 173, 4096, 0, 0, 0, 4084, 1, 0, 0, 0, 0, 2638, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 1445, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3940, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 8192, 2890, 8192, 8192, 7880] dup_rows_per_dst=[346, 5260, 0, 14, 0, 0, 0, 0] combine_rows_per_src=[8192, 8186, 8192, 8192, 2890, 8192, 8192, 7880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60598 payload_rows_remote=52406 dup_rows=4938 dup_rows_remote=3682 dedup_ratio=0.075348 router_tokens_per_expert=[0, 0, 0, 212, 4096, 0, 0, 0, 4072, 7, 0, 0, 4, 0, 2146, 0, 3, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4095, 0, 0, 2204, 0, 0, 0, 0, 0, 3, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4089, 0, 0, 0, 5, 3, 3638, 0, 0, 6] payload_rows_per_dst=[8192, 8188, 8172, 7972, 4408, 8192, 8180, 7294] dup_rows_per_dst=[1256, 3624, 38, 0, 6, 2, 2, 10] combine_rows_per_src=[8192, 8188, 8172, 7972, 4408, 8192, 8180, 7294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=377896960 dispatch_meta_bytes=229368 combine_payload_bytes=377896960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=378126328 ingress_bytes=487997440 link_bytes=487997440 token_bytes=756023288 weight_movement_bytes=110100480 total_nvlink_bytes=866123768 weight_to_token_ratio=0.145631 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.487078099e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=389480448 dispatch_meta_bytes=229392 combine_payload_bytes=389480448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=389709840 ingress_bytes=499580928 link_bytes=499580928 token_bytes=779190288 weight_movement_bytes=110100480 total_nvlink_bytes=889290768 weight_to_token_ratio=0.141301 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688534007e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=384634880 dispatch_meta_bytes=215904 combine_payload_bytes=384634880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=384850784 ingress_bytes=494735360 link_bytes=494735360 token_bytes=769485664 weight_movement_bytes=110100480 total_nvlink_bytes=879586144 weight_to_token_ratio=0.143083 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.604261610e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=378054656 dispatch_meta_bytes=229176 combine_payload_bytes=378054656 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=378283832 ingress_bytes=818456576 link_bytes=818456576 token_bytes=756338488 weight_movement_bytes=440401920 total_nvlink_bytes=1196740408 weight_to_token_ratio=0.582282 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.423430599e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60260 payload_rows_remote=52072 dup_rows=5276 dup_rows_remote=5276 dedup_ratio=0.080505 router_tokens_per_expert=[0, 0, 0, 234, 4095, 0, 0, 0, 4037, 6, 0, 0, 0, 0, 2392, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4093, 0, 1, 1907, 0, 1, 0, 1, 2, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 14, 2, 3690, 0, 0, 0] payload_rows_per_dst=[8192, 8094, 8188, 8192, 3818, 8188, 8186, 7402] dup_rows_per_dst=[466, 4776, 0, 12, 12, 0, 0, 10] combine_rows_per_src=[8192, 8094, 8188, 8192, 3818, 8188, 8186, 7402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=370800640 dispatch_meta_bytes=208360 combine_payload_bytes=370800640 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=371009000 ingress_bytes=701102080 link_bytes=701102080 token_bytes=741809640 weight_movement_bytes=330301440 total_nvlink_bytes=1072111080 weight_to_token_ratio=0.445264 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.219331829e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58964 payload_rows_remote=50938 dup_rows=6572 dup_rows_remote=5524 dedup_ratio=0.100281 router_tokens_per_expert=[0, 1877, 0, 3478, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3010, 0, 0, 0, 0, 6, 1, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 0, 0, 0, 0, 1, 4096, 0, 35, 0, 0, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 238, 4, 0, 4005, 0, 0, 0, 0, 0] payload_rows_per_dst=[3758, 8192, 6408, 8184, 8172, 8192, 8032, 8026] dup_rows_per_dst=[14, 0, 232, 8, 0, 78, 5192, 1048] combine_rows_per_src=[3758, 8192, 6408, 8184, 8172, 8192, 8032, 8026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=375646208 dispatch_meta_bytes=224352 combine_payload_bytes=375646208 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=375870560 ingress_bytes=595847168 link_bytes=595847168 token_bytes=751516768 weight_movement_bytes=220200960 total_nvlink_bytes=971717728 weight_to_token_ratio=0.293009 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.036276225e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61568 payload_rows_remote=53376 dup_rows=3968 dup_rows_remote=3966 dedup_ratio=0.060547 router_tokens_per_expert=[0, 1093, 0, 3749, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0, 0, 0, 3588, 0, 0, 0, 0, 15, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 3688, 0, 0, 0, 0, 0, 0, 196, 2, 0, 4049, 0, 0, 0, 3, 0] payload_rows_per_dst=[5554, 8192, 7636, 8192, 8184, 8192, 7514, 8104] dup_rows_per_dst=[776, 4, 2924, 4, 0, 2, 254, 4] combine_rows_per_src=[5554, 8192, 7636, 8192, 8184, 8192, 7514, 8104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62608 payload_rows_remote=54418 dup_rows=2928 dup_rows_remote=2928 dedup_ratio=0.044678 router_tokens_per_expert=[0, 1985, 0, 3235, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3458, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 3293, 0, 0, 0, 0, 0, 0, 364, 0, 0, 4033, 0, 0, 0, 0, 0] payload_rows_per_dst=[7992, 8192, 6944, 8192, 8190, 8192, 6840, 8066] dup_rows_per_dst=[2448, 0, 0, 0, 0, 6, 474, 0] combine_rows_per_src=[7992, 8192, 6944, 8192, 8190, 8192, 6840, 8066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58730 payload_rows_remote=51454 dup_rows=6806 dup_rows_remote=6606 dedup_ratio=0.103851 router_tokens_per_expert=[0, 1229, 0, 3611, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3760, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 1, 0, 3545, 0, 0, 0, 0, 0, 1, 192, 0, 0, 4027, 0, 0, 2, 0, 0] payload_rows_per_dst=[2462, 8188, 8184, 8188, 8182, 8192, 7276, 8058] dup_rows_per_dst=[0, 0, 6598, 0, 0, 8, 200, 0] combine_rows_per_src=[2462, 8188, 8184, 8188, 8182, 8192, 7276, 8058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62938 payload_rows_remote=54746 dup_rows=2598 dup_rows_remote=2596 dedup_ratio=0.039642 router_tokens_per_expert=[0, 1926, 0, 3343, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 5, 1, 0, 0, 0, 0, 3652, 0, 0, 0, 0, 0, 0, 114, 0, 0, 4068, 0, 0, 0, 1, 0] payload_rows_per_dst=[8092, 8190, 6548, 8192, 8192, 8192, 7396, 8136] dup_rows_per_dst=[2446, 0, 0, 2, 0, 12, 136, 2] combine_rows_per_src=[8092, 8190, 6548, 8192, 8192, 8192, 7396, 8136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=373252096 dispatch_meta_bytes=229392 combine_payload_bytes=373252096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373481488 ingress_bytes=483352576 link_bytes=483352576 token_bytes=746733584 weight_movement_bytes=110100480 total_nvlink_bytes=856834064 weight_to_token_ratio=0.147443 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.406296275e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=365123584 dispatch_meta_bytes=225848 combine_payload_bytes=365123584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=365349432 ingress_bytes=475224064 link_bytes=475224064 token_bytes=730473016 weight_movement_bytes=110100480 total_nvlink_bytes=840573496 weight_to_token_ratio=0.150725 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.264928082e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=382599168 dispatch_meta_bytes=229368 combine_payload_bytes=382599168 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=382828536 ingress_bytes=712900608 link_bytes=712900608 token_bytes=765427704 weight_movement_bytes=330301440 total_nvlink_bytes=1095729144 weight_to_token_ratio=0.431525 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.239851409e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=390068224 dispatch_meta_bytes=229384 combine_payload_bytes=390068224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=390297608 ingress_bytes=500168704 link_bytes=500168704 token_bytes=780365832 weight_movement_bytes=110100480 total_nvlink_bytes=890466312 weight_to_token_ratio=0.141088 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.698756399e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=368822272 dispatch_meta_bytes=232240 combine_payload_bytes=368822272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369054512 ingress_bytes=478922752 link_bytes=478922752 token_bytes=737876784 weight_movement_bytes=110100480 total_nvlink_bytes=847977264 weight_to_token_ratio=0.149213 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.329254350e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=392419328 dispatch_meta_bytes=229368 combine_payload_bytes=392419328 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=392648696 ingress_bytes=612620288 link_bytes=612620288 token_bytes=785068024 weight_movement_bytes=220200960 total_nvlink_bytes=1005268984 weight_to_token_ratio=0.280486 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.065447440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63048 payload_rows_remote=54856 dup_rows=2488 dup_rows_remote=2488 dedup_ratio=0.037964 router_tokens_per_expert=[0, 1630, 0, 3556, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3536, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 3486, 0, 0, 0, 0, 0, 0, 167, 0, 0, 4001, 0, 0, 0, 0, 0] payload_rows_per_dst=[8094, 8192, 7082, 8192, 8190, 8192, 7104, 8002] dup_rows_per_dst=[2278, 0, 0, 0, 0, 8, 202, 0] combine_rows_per_src=[8094, 8192, 7082, 8192, 8190, 8192, 7104, 8002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62214 payload_rows_remote=54100 dup_rows=3322 dup_rows_remote=304 dedup_ratio=0.050690 router_tokens_per_expert=[0, 1879, 0, 3604, 0, 6, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 3131, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 3518, 0, 0, 0, 0, 0, 0, 174, 4, 0, 4062, 0, 0, 1, 0, 0] payload_rows_per_dst=[8114, 8192, 6264, 8190, 8176, 7996, 7150, 8132] dup_rows_per_dst=[3018, 4, 2, 0, 62, 0, 234, 2] combine_rows_per_src=[8114, 8192, 6264, 8190, 8176, 7996, 7150, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62446 payload_rows_remote=55732 dup_rows=3090 dup_rows_remote=3090 dedup_ratio=0.047150 router_tokens_per_expert=[0, 1797, 1, 3447, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3341, 0, 0, 0, 0, 16, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 43, 0, 0, 0, 0, 0, 3373, 0, 0, 0, 0, 0, 0, 393, 1, 1, 3976, 0, 0, 0, 0, 0] payload_rows_per_dst=[7998, 8190, 6714, 8190, 8188, 8190, 7024, 7952] dup_rows_per_dst=[2492, 0, 0, 2, 0, 84, 508, 4] combine_rows_per_src=[7998, 8190, 6714, 8190, 8188, 8190, 7024, 7952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54684 payload_rows_remote=46492 dup_rows=10852 dup_rows_remote=10806 dedup_ratio=0.165588 router_tokens_per_expert=[0, 16, 0, 2080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2395, 0, 3637, 2661, 1770, 0, 0, 0, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 1, 0, 0, 4096, 2744, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 0, 844, 4096, 0, 20, 0, 0, 0, 3, 0] payload_rows_per_dst=[4162, 4790, 6070, 7332, 7840, 8192, 8106, 8192] dup_rows_per_dst=[30, 0, 1720, 582, 996, 528, 6950, 46] combine_rows_per_src=[4162, 4790, 6070, 7332, 7840, 8192, 8106, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57004 payload_rows_remote=48870 dup_rows=8532 dup_rows_remote=4080 dedup_ratio=0.130188 router_tokens_per_expert=[0, 3, 0, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2393, 0, 3899, 2131, 1644, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3714, 4, 0, 0, 4096, 2575, 47, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 706, 4096, 0, 7, 0, 0, 0, 0, 0] payload_rows_per_dst=[5564, 4786, 6104, 7850, 8134, 8192, 8182, 8192] dup_rows_per_dst=[0, 0, 1446, 1110, 4452, 98, 1412, 14] combine_rows_per_src=[5564, 4786, 6104, 7850, 8134, 8192, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=393207808 dispatch_meta_bytes=229376 combine_payload_bytes=393207808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=393437184 ingress_bytes=503308288 link_bytes=503308288 token_bytes=786644992 weight_movement_bytes=110100480 total_nvlink_bytes=896745472 weight_to_token_ratio=0.139962 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.753358928e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56436 payload_rows_remote=48244 dup_rows=9100 dup_rows_remote=8714 dedup_ratio=0.138855 router_tokens_per_expert=[0, 4, 0, 2914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2028, 0, 3869, 1920, 2033, 0, 0, 0, 0, 592, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3693, 1, 0, 0, 4096, 2530, 188, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 3, 0, 713, 4096, 0, 32, 0, 0, 0, 0, 0] payload_rows_per_dst=[5830, 4056, 6296, 7796, 7966, 8192, 8108, 8192] dup_rows_per_dst=[6, 0, 1610, 1128, 3632, 386, 2274, 64] combine_rows_per_src=[5830, 4056, 6296, 7796, 7966, 8192, 8108, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=387788800 dispatch_meta_bytes=217616 combine_payload_bytes=387788800 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=388006416 ingress_bytes=828190720 link_bytes=828190720 token_bytes=775795216 weight_movement_bytes=440401920 total_nvlink_bytes=1216197136 weight_to_token_ratio=0.567678 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.440359876e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55930 payload_rows_remote=47756 dup_rows=9606 dup_rows_remote=2128 dedup_ratio=0.146576 router_tokens_per_expert=[0, 5, 0, 2868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2195, 0, 3899, 2498, 1181, 0, 0, 0, 0, 398, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3611, 0, 0, 0, 4095, 2874, 83, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4082, 0, 0, 0, 869, 4096, 0, 9, 0, 0, 0, 1, 0] payload_rows_per_dst=[5746, 4390, 6188, 7828, 7222, 8190, 8174, 8192] dup_rows_per_dst=[0, 0, 1170, 772, 0, 166, 7478, 20] combine_rows_per_src=[5746, 4390, 6188, 7828, 7222, 8190, 8174, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=399486976 dispatch_meta_bytes=235288 combine_payload_bytes=399486976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399722264 ingress_bytes=509587456 link_bytes=509587456 token_bytes=799209240 weight_movement_bytes=110100480 total_nvlink_bytes=909309720 weight_to_token_ratio=0.137762 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=27136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.862563987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56764 payload_rows_remote=48968 dup_rows=8772 dup_rows_remote=7808 dedup_ratio=0.133850 router_tokens_per_expert=[0, 2, 0, 2433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2430, 0, 3876, 2318, 1742, 0, 0, 0, 0, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3658, 0, 0, 0, 4096, 2666, 62, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 1, 0, 805, 4096, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[4868, 7102, 6464, 7796, 5980, 8192, 8170, 8192] dup_rows_per_dst=[2, 3428, 1656, 964, 998, 126, 1594, 4] combine_rows_per_src=[4868, 7102, 6464, 7796, 5980, 8192, 8170, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=333254656 dispatch_meta_bytes=229192 combine_payload_bytes=333254656 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=333483848 ingress_bytes=443355136 link_bytes=443355136 token_bytes=666738504 weight_movement_bytes=110100480 total_nvlink_bytes=776838984 weight_to_token_ratio=0.165133 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.710675009e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=350300160 dispatch_meta_bytes=211800 combine_payload_bytes=350300160 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=350511960 ingress_bytes=790702080 link_bytes=790702080 token_bytes=700812120 weight_movement_bytes=440401920 total_nvlink_bytes=1141214040 weight_to_token_ratio=0.628417 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=15744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.375160965e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=345812992 dispatch_meta_bytes=227832 combine_payload_bytes=345812992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=346040824 ingress_bytes=455913472 link_bytes=455913472 token_bytes=691853816 weight_movement_bytes=110100480 total_nvlink_bytes=801954296 weight_to_token_ratio=0.159138 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.929085127e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=342315008 dispatch_meta_bytes=199536 combine_payload_bytes=342315008 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=342514544 ingress_bytes=562515968 link_bytes=562515968 token_bytes=684829552 weight_movement_bytes=220200960 total_nvlink_bytes=905030512 weight_to_token_ratio=0.321541 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=20352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.783077863e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=351002624 dispatch_meta_bytes=227104 combine_payload_bytes=351002624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351229728 ingress_bytes=461103104 link_bytes=461103104 token_bytes=702232352 weight_movement_bytes=110100480 total_nvlink_bytes=812332832 weight_to_token_ratio=0.156786 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=29184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.019341363e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54218 payload_rows_remote=46256 dup_rows=11318 dup_rows_remote=7052 dedup_ratio=0.172699 router_tokens_per_expert=[0, 4, 0, 2306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2560, 0, 3955, 2295, 1849, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3554, 3, 0, 0, 4096, 2854, 64, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 0, 770, 4096, 0, 19, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 7962, 6528, 7916, 5708, 8192, 1540, 8192] dup_rows_per_dst=[4488, 4266, 1760, 622, 6, 138, 0, 38] combine_rows_per_src=[8180, 7962, 6528, 7916, 5708, 8192, 1540, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50210 payload_rows_remote=42020 dup_rows=15326 dup_rows_remote=9900 dedup_ratio=0.233856 router_tokens_per_expert=[0, 19, 0, 2767, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2277, 0, 3564, 1912, 2151, 0, 0, 0, 0, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 2, 0, 0, 4096, 2504, 237, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 756, 4096, 0, 32, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8124, 6892, 5038, 6078, 6436, 1512, 7940] dup_rows_per_dst=[5426, 4176, 3994, 548, 1028, 152, 0, 2] combine_rows_per_src=[8190, 8124, 6892, 5038, 6078, 6436, 1512, 7940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53628 payload_rows_remote=47474 dup_rows=11908 dup_rows_remote=10424 dedup_ratio=0.181702 router_tokens_per_expert=[0, 6, 0, 2549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2442, 0, 3626, 2106, 1713, 0, 0, 0, 0, 498, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3786, 8, 0, 0, 4096, 2740, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 3, 0, 923, 4096, 0, 32, 0, 0, 0, 0, 0] payload_rows_per_dst=[8130, 8110, 6154, 7392, 5482, 8192, 1976, 8192] dup_rows_per_dst=[4914, 4346, 1484, 862, 14, 164, 60, 64] combine_rows_per_src=[8130, 8110, 6154, 7392, 5482, 8192, 1976, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=331563008 dispatch_meta_bytes=213232 combine_payload_bytes=331563008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=331776240 ingress_bytes=441663488 link_bytes=441663488 token_bytes=663339248 weight_movement_bytes=110100480 total_nvlink_bytes=773439728 weight_to_token_ratio=0.165979 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=18816 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.681254468e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62404 payload_rows_remote=54634 dup_rows=3132 dup_rows_remote=3126 dedup_ratio=0.047791 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3392, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 0, 3, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3445, 0, 0, 0, 1560, 0, 0, 0, 0, 3] payload_rows_per_dst=[8192, 6784, 8192, 8192, 7770, 8192, 6890, 8192] dup_rows_per_dst=[0, 0, 0, 0, 6, 0, 0, 3126] combine_rows_per_src=[8192, 6784, 8192, 8192, 7770, 8192, 6890, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62354 payload_rows_remote=54162 dup_rows=3182 dup_rows_remote=118 dedup_ratio=0.048553 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 3681, 0, 4095, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3507, 0, 121, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 3454, 0, 0, 0, 1525, 0, 0, 0, 0, 7] payload_rows_per_dst=[8174, 7362, 8192, 8190, 7144, 8190, 6910, 8192] dup_rows_per_dst=[0, 0, 0, 2, 114, 0, 2, 3064] combine_rows_per_src=[8174, 7362, 8192, 8190, 7144, 8190, 6910, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61604 payload_rows_remote=53412 dup_rows=3932 dup_rows_remote=3930 dedup_ratio=0.059998 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 3424, 0, 4096, 0, 4095, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3559, 0, 36, 4096, 0, 4093, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3321, 0, 0, 0, 1942, 0, 0, 0, 0, 4] payload_rows_per_dst=[8188, 6848, 8192, 8192, 7158, 8192, 6642, 8192] dup_rows_per_dst=[0, 0, 6, 0, 32, 2, 0, 3892] combine_rows_per_src=[8188, 6848, 8192, 8192, 7158, 8192, 6642, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64168 payload_rows_remote=56520 dup_rows=1368 dup_rows_remote=1368 dedup_ratio=0.020874 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 3902, 0, 4096, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 31, 4096, 0, 4092, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3824, 0, 0, 0, 652, 0, 0, 0, 0, 3] payload_rows_per_dst=[8186, 7806, 8190, 8192, 7762, 8192, 7648, 8192] dup_rows_per_dst=[0, 0, 0, 0, 54, 4, 0, 1310] combine_rows_per_src=[8186, 7806, 8190, 8192, 7762, 8192, 7648, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=301199360 dispatch_meta_bytes=207680 combine_payload_bytes=301199360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=301407040 ingress_bytes=411299840 link_bytes=411299840 token_bytes=602606400 weight_movement_bytes=110100480 total_nvlink_bytes=712706880 weight_to_token_ratio=0.182707 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.153180690e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=340293632 dispatch_meta_bytes=231592 combine_payload_bytes=340293632 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=340525224 ingress_bytes=670595072 link_bytes=670595072 token_bytes=680818856 weight_movement_bytes=330301440 total_nvlink_bytes=1011120296 weight_to_token_ratio=0.485153 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.166275124e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62124 payload_rows_remote=53932 dup_rows=3412 dup_rows_remote=3412 dedup_ratio=0.052063 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3681, 0, 4094, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3716, 0, 34, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3179, 0, 0, 0, 1679, 0, 0, 0, 0, 1] payload_rows_per_dst=[8190, 7362, 8192, 8192, 7450, 8188, 6358, 8192] dup_rows_per_dst=[0, 0, 2, 0, 50, 0, 0, 3360] combine_rows_per_src=[8190, 7362, 8192, 8192, 7450, 8188, 6358, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=391616512 dispatch_meta_bytes=231040 combine_payload_bytes=391616512 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=391847552 ingress_bytes=832018432 link_bytes=832018432 token_bytes=783464064 weight_movement_bytes=440401920 total_nvlink_bytes=1223865984 weight_to_token_ratio=0.562121 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.447016897e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=388233216 dispatch_meta_bytes=217120 combine_payload_bytes=388233216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=388450336 ingress_bytes=498333696 link_bytes=498333696 token_bytes=776683552 weight_movement_bytes=110100480 total_nvlink_bytes=886784032 weight_to_token_ratio=0.141757 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.666842592e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=405135360 dispatch_meta_bytes=231552 combine_payload_bytes=405135360 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=405366912 ingress_bytes=735436800 link_bytes=735436800 token_bytes=810502272 weight_movement_bytes=330301440 total_nvlink_bytes=1140803712 weight_to_token_ratio=0.407527 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.279045554e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=382857216 dispatch_meta_bytes=229368 combine_payload_bytes=382857216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383086584 ingress_bytes=492957696 link_bytes=492957696 token_bytes=765943800 weight_movement_bytes=110100480 total_nvlink_bytes=876044280 weight_to_token_ratio=0.143745 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.573345110e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=386584576 dispatch_meta_bytes=229376 combine_payload_bytes=386584576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=386813952 ingress_bytes=496685056 link_bytes=496685056 token_bytes=773398528 weight_movement_bytes=110100480 total_nvlink_bytes=883499008 weight_to_token_ratio=0.142359 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.638170030e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58944 payload_rows_remote=51348 dup_rows=6592 dup_rows_remote=5694 dedup_ratio=0.100586 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3677, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 0, 26, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3552, 0, 0, 0, 1269, 0, 0, 0, 0, 15] payload_rows_per_dst=[8186, 7596, 8192, 8192, 7856, 8192, 8192, 2538] dup_rows_per_dst=[0, 898, 8, 0, 5656, 0, 0, 30] combine_rows_per_src=[8186, 7596, 8192, 8192, 7856, 8192, 8192, 2538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59580 payload_rows_remote=51394 dup_rows=5956 dup_rows_remote=5914 dedup_ratio=0.090881 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 3407, 0, 4095, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 0, 34, 4096, 0, 4091, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 3596, 0, 0, 0, 1823, 0, 0, 0, 0, 7] payload_rows_per_dst=[8186, 8190, 8178, 8192, 6804, 8192, 8192, 3646] dup_rows_per_dst=[42, 5842, 0, 0, 46, 0, 12, 14] combine_rows_per_src=[8186, 8190, 8178, 8192, 6804, 8192, 8192, 3646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58760 payload_rows_remote=50570 dup_rows=6776 dup_rows_remote=6774 dedup_ratio=0.103394 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 0, 0, 0, 3424, 0, 4096, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3701, 0, 14, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3537, 0, 0, 0, 1611, 0, 0, 0, 0, 1] payload_rows_per_dst=[8186, 6848, 8190, 8192, 7528, 8192, 8140, 3484] dup_rows_per_dst=[6, 0, 2, 0, 106, 0, 6550, 112] combine_rows_per_src=[8186, 6848, 8190, 8192, 7528, 8192, 8140, 3484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=368062464 dispatch_meta_bytes=228168 combine_payload_bytes=368062464 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=368290632 ingress_bytes=698363904 link_bytes=698363904 token_bytes=736353096 weight_movement_bytes=330301440 total_nvlink_bytes=1066654536 weight_to_token_ratio=0.448564 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.214569690e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=368392192 dispatch_meta_bytes=229232 combine_payload_bytes=368392192 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=368621424 ingress_bytes=588593152 link_bytes=588593152 token_bytes=737013616 weight_movement_bytes=220200960 total_nvlink_bytes=957214576 weight_to_token_ratio=0.298775 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.023660298e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62400 payload_rows_remote=54212 dup_rows=3136 dup_rows_remote=3118 dedup_ratio=0.047852 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 653, 4096, 0, 4096, 0, 2084, 0, 0, 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 270, 0, 0, 0, 0, 0, 0, 642, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8190, 5432, 8192, 7840, 8192, 8174, 8192, 8188] dup_rows_per_dst=[2, 44, 1244, 0, 0, 540, 1288, 18] combine_rows_per_src=[8190, 5432, 8192, 7840, 8192, 8174, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=362485760 dispatch_meta_bytes=229376 combine_payload_bytes=362485760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362715136 ingress_bytes=472586240 link_bytes=472586240 token_bytes=725200896 weight_movement_bytes=110100480 total_nvlink_bytes=835301376 weight_to_token_ratio=0.151821 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.219051984e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64224 payload_rows_remote=56032 dup_rows=1312 dup_rows_remote=1312 dedup_ratio=0.020020 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 546, 4096, 0, 4096, 0, 3196, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4071, 39, 0, 0, 0, 0, 0, 0, 333, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7344, 8192, 7778, 8192, 8142, 8192, 8192] dup_rows_per_dst=[0, 140, 428, 0, 0, 78, 666, 0] combine_rows_per_src=[8192, 7344, 8192, 7778, 8192, 8142, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62744 payload_rows_remote=54552 dup_rows=2792 dup_rows_remote=1212 dedup_ratio=0.042603 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 585, 4096, 0, 4096, 0, 2396, 0, 0, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3933, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4031, 147, 0, 0, 0, 0, 0, 0, 788, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5860, 8192, 7868, 8192, 8062, 8192, 8186] dup_rows_per_dst=[0, 102, 814, 2, 0, 294, 1580, 0] combine_rows_per_src=[8192, 5860, 8192, 7868, 8192, 8062, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62264 payload_rows_remote=54220 dup_rows=3272 dup_rows_remote=2872 dedup_ratio=0.049927 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 785, 4096, 0, 4096, 0, 1982, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4020, 202, 0, 0, 0, 0, 0, 0, 938, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 5390, 8192, 7872, 8192, 8044, 8192, 8190] dup_rows_per_dst=[0, 144, 850, 0, 0, 400, 1878, 0] combine_rows_per_src=[8192, 5390, 8192, 7872, 8192, 8044, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62538 payload_rows_remote=54672 dup_rows=2998 dup_rows_remote=2998 dedup_ratio=0.045746 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 684, 4096, 0, 4096, 0, 2155, 0, 0, 0, 542, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4056, 224, 0, 0, 0, 0, 0, 0, 693, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5596, 8192, 7866, 8192, 8116, 8192, 8192] dup_rows_per_dst=[0, 82, 1084, 0, 0, 444, 1388, 0] combine_rows_per_src=[8192, 5596, 8192, 7866, 8192, 8116, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=388591616 dispatch_meta_bytes=229320 combine_payload_bytes=388591616 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=388820936 ingress_bytes=608792576 link_bytes=608792576 token_bytes=777412552 weight_movement_bytes=220200960 total_nvlink_bytes=997613512 weight_to_token_ratio=0.283249 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058790419e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=401637376 dispatch_meta_bytes=229376 combine_payload_bytes=401637376 prefetch_bytes=0 reduce_bytes=0 egress_bytes=401866752 ingress_bytes=401637376 link_bytes=401866752 token_bytes=803504128 weight_movement_bytes=0 total_nvlink_bytes=803504128 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.989123775e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=391028736 dispatch_meta_bytes=223056 combine_payload_bytes=391028736 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=391251792 ingress_bytes=611229696 link_bytes=611229696 token_bytes=782280528 weight_movement_bytes=220200960 total_nvlink_bytes=1002481488 weight_to_token_ratio=0.281486 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.063028971e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=388648960 dispatch_meta_bytes=228368 combine_payload_bytes=388648960 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=388877328 ingress_bytes=608849920 link_bytes=608849920 token_bytes=777526288 weight_movement_bytes=220200960 total_nvlink_bytes=997727248 weight_to_token_ratio=0.283207 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058890149e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60896 payload_rows_remote=54326 dup_rows=4640 dup_rows_remote=2302 dedup_ratio=0.070801 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 927, 4096, 0, 4096, 0, 2125, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4045, 257, 0, 0, 0, 0, 0, 0, 594, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6570, 8192, 7998, 8192, 8092, 5468, 8192] dup_rows_per_dst=[0, 2338, 682, 0, 0, 762, 858, 0] combine_rows_per_src=[8192, 6570, 8192, 7998, 8192, 8092, 5468, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=391888896 dispatch_meta_bytes=230680 combine_payload_bytes=391888896 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=392119576 ingress_bytes=722190336 link_bytes=722190336 token_bytes=784008472 weight_movement_bytes=330301440 total_nvlink_bytes=1114309912 weight_to_token_ratio=0.421298 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.256007774e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50028 payload_rows_remote=41836 dup_rows=15508 dup_rows_remote=15508 dedup_ratio=0.236633 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 607, 4096, 0, 4096, 0, 2201, 0, 0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3889, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 344, 0, 0, 0, 0, 0, 0, 761, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 2182, 7926, 8192, 5636, 1522, 8186] dup_rows_per_dst=[0, 12238, 206, 2556, 0, 506, 2, 0] combine_rows_per_src=[8192, 8192, 2182, 7926, 8192, 5636, 1522, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62844 payload_rows_remote=54652 dup_rows=2692 dup_rows_remote=1990 dedup_ratio=0.041077 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 709, 4096, 0, 4096, 0, 2323, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4056, 67, 0, 0, 0, 0, 0, 0, 867, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5942, 8192, 7830, 8192, 8112, 8192, 8192] dup_rows_per_dst=[0, 122, 702, 0, 0, 134, 1734, 0] combine_rows_per_src=[8192, 5942, 8192, 7830, 8192, 8112, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=389408768 dispatch_meta_bytes=226512 combine_payload_bytes=389408768 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=389635280 ingress_bytes=719710208 link_bytes=719710208 token_bytes=779044048 weight_movement_bytes=330301440 total_nvlink_bytes=1109345488 weight_to_token_ratio=0.423983 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=18560 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251694424e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=299880448 dispatch_meta_bytes=229376 combine_payload_bytes=299880448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=300109824 ingress_bytes=409980928 link_bytes=409980928 token_bytes=599990272 weight_movement_bytes=110100480 total_nvlink_bytes=710090752 weight_to_token_ratio=0.183504 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.130242641e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58966 payload_rows_remote=50794 dup_rows=6570 dup_rows_remote=6458 dedup_ratio=0.100250 router_tokens_per_expert=[0, 4029, 0, 0, 4043, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 43, 1006, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3158, 2, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086] payload_rows_per_dst=[7874, 8080, 2098, 8192, 8192, 8166, 8192, 8172] dup_rows_per_dst=[0, 214, 0, 0, 0, 6244, 0, 112] combine_rows_per_src=[7874, 8080, 2098, 8192, 8192, 8166, 8192, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=391745536 dispatch_meta_bytes=226568 combine_payload_bytes=391745536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=391972104 ingress_bytes=501846016 link_bytes=501846016 token_bytes=783717640 weight_movement_bytes=110100480 total_nvlink_bytes=893818120 weight_to_token_ratio=0.140485 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.727927613e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64922 payload_rows_remote=56730 dup_rows=614 dup_rows_remote=614 dedup_ratio=0.009369 router_tokens_per_expert=[0, 4095, 0, 0, 4090, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 43, 262, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3797, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8180, 8190, 8192, 8192, 8192, 7594, 8192, 8190] dup_rows_per_dst=[0, 2, 610, 2, 0, 0, 0, 0] combine_rows_per_src=[8180, 8190, 8192, 8192, 8192, 7594, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62780 payload_rows_remote=54588 dup_rows=2756 dup_rows_remote=2752 dedup_ratio=0.042053 router_tokens_per_expert=[0, 4095, 0, 0, 4078, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 42, 525, 0, 4095, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 2, 1, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4088] payload_rows_per_dst=[8156, 8190, 6272, 8190, 8192, 7410, 8192, 8178] dup_rows_per_dst=[0, 8, 816, 2, 0, 1926, 4, 0] combine_rows_per_src=[8156, 8190, 6272, 8190, 8192, 7410, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64194 payload_rows_remote=57238 dup_rows=1342 dup_rows_remote=1330 dedup_ratio=0.020477 router_tokens_per_expert=[0, 4085, 3, 0, 4066, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 34, 632, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3478, 3, 3, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4082] payload_rows_per_dst=[8138, 8172, 8192, 8188, 8192, 6956, 8192, 8164] dup_rows_per_dst=[0, 2, 1326, 0, 0, 12, 2, 0] combine_rows_per_src=[8138, 8172, 8192, 8188, 8192, 6956, 8192, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63978 payload_rows_remote=55788 dup_rows=1558 dup_rows_remote=1556 dedup_ratio=0.023773 router_tokens_per_expert=[0, 4078, 1, 0, 4051, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 42, 734, 0, 4095, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3381, 1, 1, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4093] payload_rows_per_dst=[8104, 8158, 8192, 8190, 8192, 6764, 8192, 8186] dup_rows_per_dst=[0, 2, 1550, 2, 0, 2, 2, 0] combine_rows_per_src=[8104, 8158, 8192, 8190, 8192, 6764, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=364091392 dispatch_meta_bytes=229008 combine_payload_bytes=364091392 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=364320400 ingress_bytes=694392832 link_bytes=694392832 token_bytes=728411792 weight_movement_bytes=330301440 total_nvlink_bytes=1058713232 weight_to_token_ratio=0.453454 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.207663343e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=406640640 dispatch_meta_bytes=229376 combine_payload_bytes=406640640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406870016 ingress_bytes=516741120 link_bytes=516741120 token_bytes=813510656 weight_movement_bytes=110100480 total_nvlink_bytes=923611136 weight_to_token_ratio=0.135340 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.986977970e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=391286784 dispatch_meta_bytes=229360 combine_payload_bytes=391286784 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=391516144 ingress_bytes=501387264 link_bytes=501387264 token_bytes=782802928 weight_movement_bytes=110100480 total_nvlink_bytes=892903408 weight_to_token_ratio=0.140649 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.719949161e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=410281984 dispatch_meta_bytes=234272 combine_payload_bytes=410281984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410516256 ingress_bytes=520382464 link_bytes=520382464 token_bytes=820798240 weight_movement_bytes=110100480 total_nvlink_bytes=930898720 weight_to_token_ratio=0.134138 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=27648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.050306931e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=399888384 dispatch_meta_bytes=229376 combine_payload_bytes=399888384 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=400117760 ingress_bytes=620089344 link_bytes=620089344 token_bytes=800006144 weight_movement_bytes=220200960 total_nvlink_bytes=1020207104 weight_to_token_ratio=0.275249 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.078437356e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64214 payload_rows_remote=56050 dup_rows=1322 dup_rows_remote=1322 dedup_ratio=0.020172 router_tokens_per_expert=[0, 4082, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 607, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3509, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094] payload_rows_per_dst=[8076, 8164, 8192, 8192, 8192, 7018, 8192, 8188] dup_rows_per_dst=[0, 0, 1320, 2, 0, 0, 0, 0] combine_rows_per_src=[8076, 8164, 8192, 8192, 8192, 7018, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63206 payload_rows_remote=55026 dup_rows=2330 dup_rows_remote=1628 dedup_ratio=0.035553 router_tokens_per_expert=[0, 4074, 0, 0, 4066, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 45, 942, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3156, 2, 2, 0, 7, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 4086] payload_rows_per_dst=[8180, 8154, 7824, 8188, 8192, 6316, 8182, 8170] dup_rows_per_dst=[702, 6, 1614, 4, 0, 4, 0, 0] combine_rows_per_src=[8180, 8154, 7824, 8188, 8192, 6316, 8182, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64370 payload_rows_remote=56180 dup_rows=1166 dup_rows_remote=18 dedup_ratio=0.017792 router_tokens_per_expert=[0, 4091, 1, 0, 4062, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 40, 536, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 3, 1, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088] payload_rows_per_dst=[8126, 8186, 8190, 8192, 8192, 7116, 8192, 8176] dup_rows_per_dst=[0, 8, 1148, 0, 0, 8, 2, 0] combine_rows_per_src=[8126, 8186, 8190, 8192, 8192, 7116, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=401766400 dispatch_meta_bytes=229488 combine_payload_bytes=401766400 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=401995888 ingress_bytes=621967360 link_bytes=621967360 token_bytes=803762288 weight_movement_bytes=220200960 total_nvlink_bytes=1023963248 weight_to_token_ratio=0.273963 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.081703535e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65468 payload_rows_remote=57280 dup_rows=68 dup_rows_remote=58 dedup_ratio=0.001038 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4093, 3, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 2, 0, 4094, 0, 0] payload_rows_per_dst=[8180, 8192, 8182, 8188, 8166, 8192, 8180, 8188] dup_rows_per_dst=[2, 0, 2, 10, 12, 0, 32, 10] combine_rows_per_src=[8180, 8192, 8182, 8188, 8166, 8192, 8180, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=394426368 dispatch_meta_bytes=226616 combine_payload_bytes=394426368 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=394652984 ingress_bytes=724727808 link_bytes=724727808 token_bytes=789079352 weight_movement_bytes=330301440 total_nvlink_bytes=1119380792 weight_to_token_ratio=0.418591 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.260420855e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=402698240 dispatch_meta_bytes=224792 combine_payload_bytes=402698240 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=402923032 ingress_bytes=953200640 link_bytes=953200640 token_bytes=805621272 weight_movement_bytes=550502400 total_nvlink_bytes=1356123672 weight_to_token_ratio=0.683327 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=27264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.657772688e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65522 payload_rows_remote=57334 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 3, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8186, 8192, 8192, 8188, 8190] dup_rows_per_dst=[4, 0, 0, 0, 10, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8186, 8192, 8192, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 2, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 4, 0, 4094, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8184, 8192, 8192, 8180, 8188] dup_rows_per_dst=[4, 0, 2, 0, 10, 0, 0, 8] combine_rows_per_src=[8192, 8192, 8192, 8184, 8192, 8192, 8180, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65530 payload_rows_remote=57340 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8190, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 2, 0, 4, 0, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8190, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410583040 dispatch_meta_bytes=229352 combine_payload_bytes=410583040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410812392 ingress_bytes=520683520 link_bytes=520683520 token_bytes=821395432 weight_movement_bytes=110100480 total_nvlink_bytes=931495912 weight_to_token_ratio=0.134041 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.055542790e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=229376 combine_payload_bytes=411041792 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=411271168 ingress_bytes=741343232 link_bytes=741343232 token_bytes=822312960 weight_movement_bytes=330301440 total_nvlink_bytes=1152614400 weight_to_token_ratio=0.401674 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289317810e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410970112 dispatch_meta_bytes=229392 combine_payload_bytes=410970112 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411199504 ingress_bytes=741271552 link_bytes=741271552 token_bytes=822169616 weight_movement_bytes=330301440 total_nvlink_bytes=1152471056 weight_to_token_ratio=0.401744 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289193147e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410869760 dispatch_meta_bytes=229376 combine_payload_bytes=410869760 prefetch_bytes=0 reduce_bytes=0 egress_bytes=411099136 ingress_bytes=410869760 link_bytes=411099136 token_bytes=821968896 weight_movement_bytes=0 total_nvlink_bytes=821968896 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.149690117e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=411013120 dispatch_meta_bytes=229384 combine_payload_bytes=411013120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=411242504 ingress_bytes=631214080 link_bytes=631214080 token_bytes=822255624 weight_movement_bytes=220200960 total_nvlink_bytes=1042456584 weight_to_token_ratio=0.267801 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097785102e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65512 payload_rows_remote=57320 dup_rows=24 dup_rows_remote=20 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8174, 8192, 8192, 8188] dup_rows_per_dst=[4, 0, 2, 18, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8174, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=12 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 3, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 4093, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8192, 8192, 8188, 8186] dup_rows_per_dst=[0, 0, 4, 0, 8, 0, 0, 4] combine_rows_per_src=[8192, 8192, 8192, 8186, 8192, 8192, 8188, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=229376 combine_payload_bytes=411041792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411271168 ingress_bytes=521142272 link_bytes=521142272 token_bytes=822312960 weight_movement_bytes=110100480 total_nvlink_bytes=932413440 weight_to_token_ratio=0.133891 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.063521242e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63866 payload_rows_remote=55674 dup_rows=1670 dup_rows_remote=1670 dedup_ratio=0.025482 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4090, 0, 41, 0, 0, 0, 0, 0, 0, 3503, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 551, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8158, 7738, 7012, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 10, 74, 28, 1556, 0, 0] combine_rows_per_src=[8192, 8190, 8158, 7738, 7012, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64196 payload_rows_remote=57344 dup_rows=1340 dup_rows_remote=1340 dedup_ratio=0.020447 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 37, 0, 0, 0, 0, 0, 0, 3426, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6852, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 74, 0, 1266, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 6852, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410869760 dispatch_meta_bytes=229360 combine_payload_bytes=410869760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411099120 ingress_bytes=520970240 link_bytes=520970240 token_bytes=821968880 weight_movement_bytes=110100480 total_nvlink_bytes=932069360 weight_to_token_ratio=0.133947 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.060529322e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410927104 dispatch_meta_bytes=229360 combine_payload_bytes=410927104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411156464 ingress_bytes=521027584 link_bytes=521027584 token_bytes=822083568 weight_movement_bytes=110100480 total_nvlink_bytes=932184048 weight_to_token_ratio=0.133929 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.061526629e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63988 payload_rows_remote=55796 dup_rows=1548 dup_rows_remote=1546 dedup_ratio=0.023621 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4094, 0, 24, 0, 0, 0, 0, 0, 0, 3324, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 746, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 6648, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 48, 0, 1492, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 6648, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64306 payload_rows_remote=56114 dup_rows=1230 dup_rows_remote=36 dedup_ratio=0.018768 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 15, 0, 0, 0, 0, 0, 0, 3484, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 596, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 6968, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 30, 0, 1194, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 6968, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64204 payload_rows_remote=56012 dup_rows=1332 dup_rows_remote=1276 dedup_ratio=0.020325 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 28, 0, 0, 0, 0, 0, 0, 3430, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6860, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 56, 0, 1274, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 6860, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=399071232 dispatch_meta_bytes=229376 combine_payload_bytes=399071232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399300608 ingress_bytes=509171712 link_bytes=509171712 token_bytes=798371840 weight_movement_bytes=110100480 total_nvlink_bytes=908472320 weight_to_token_ratio=0.137906 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.855333515e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=234736 combine_payload_bytes=411041792 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411276528 ingress_bytes=741343232 link_bytes=741343232 token_bytes=822318320 weight_movement_bytes=330301440 total_nvlink_bytes=1152619760 weight_to_token_ratio=0.401671 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289317810e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61446 payload_rows_remote=53254 dup_rows=4090 dup_rows_remote=4090 dedup_ratio=0.062408 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 3368, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 708, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7156, 5138, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 38, 3048, 1004, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7156, 5138, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=399945728 dispatch_meta_bytes=229368 combine_payload_bytes=399945728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400175096 ingress_bytes=510046208 link_bytes=510046208 token_bytes=800120824 weight_movement_bytes=110100480 total_nvlink_bytes=910221304 weight_to_token_ratio=0.137605 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.870542439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=402225152 dispatch_meta_bytes=224600 combine_payload_bytes=402225152 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=402449752 ingress_bytes=732526592 link_bytes=732526592 token_bytes=804674904 weight_movement_bytes=330301440 total_nvlink_bytes=1134976344 weight_to_token_ratio=0.410478 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=27008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.273984223e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=401494016 dispatch_meta_bytes=229152 combine_payload_bytes=401494016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=401723168 ingress_bytes=621694976 link_bytes=621694976 token_bytes=803217184 weight_movement_bytes=220200960 total_nvlink_bytes=1023418144 weight_to_token_ratio=0.274149 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.081229815e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7204737856 prefetch_bytes=1101004800 reduce_bytes=990904320 weight_movement_bytes=2091909120 egress_bytes=3603477312 ingress_bytes=5693169664 link_bytes=5693169664 total_nvlink_bytes=9296646976 weight_to_token_ratio=0.290352 prefetch_share_of_ingress=0.193390 dedup_ratio=0.111160 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=9 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=6731 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.901358411e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7641188368 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=3821733904 ingress_bytes=6021464064 link_bytes=6021693440 total_nvlink_bytes=9843197968 weight_to_token_ratio=0.288176 prefetch_share_of_ingress=0.164562 dedup_ratio=0.071228 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=6985 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.047271529e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58488 payload_rows_remote=50296 dup_rows=7048 dup_rows_remote=7048 dedup_ratio=0.107544 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 42, 0, 0, 0, 0, 0, 0, 3480, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 1150, 8190, 8192] dup_rows_per_dst=[0, 0, 4, 84, 6960, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 1150, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63150 payload_rows_remote=54958 dup_rows=2386 dup_rows_remote=2380 dedup_ratio=0.036407 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 35, 0, 0, 0, 0, 0, 0, 2906, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 5812, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 70, 0, 2310, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 5812, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7489142328 prefetch_bytes=660602880 reduce_bytes=1211105280 weight_movement_bytes=1871708160 egress_bytes=3745697336 ingress_bytes=5615153152 link_bytes=5615153152 total_nvlink_bytes=9360850488 weight_to_token_ratio=0.249923 prefetch_share_of_ingress=0.117646 dedup_ratio=0.084360 prefetch_slots=12 unused_prefetch_slots=148 slot_utilization=0.075000 reduce_slots=11 remote_demand=6 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048529 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.765674865e-05 prefetch_bytes_per_gemm_flop=1.148897059e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7520487072 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3761387168 ingress_bytes=5630808064 link_bytes=5631037440 total_nvlink_bytes=9392195232 weight_to_token_ratio=0.248881 prefetch_share_of_ingress=0.175979 dedup_ratio=0.079892 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=7239 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.793300255e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=381724672 dispatch_meta_bytes=229376 combine_payload_bytes=381724672 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=381954048 ingress_bytes=491825152 link_bytes=491825152 token_bytes=763678720 weight_movement_bytes=110100480 total_nvlink_bytes=873779200 weight_to_token_ratio=0.144171 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.553648307e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7531850864 prefetch_bytes=880803840 reduce_bytes=1211105280 weight_movement_bytes=2091909120 egress_bytes=3767073904 ingress_bytes=5856686080 link_bytes=5856686080 total_nvlink_bytes=9623759984 weight_to_token_ratio=0.277742 prefetch_share_of_ingress=0.150393 dedup_ratio=0.079065 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=11 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047794 gemm_inner_pad_upper_bound=6858 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.018574035e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=360521728 dispatch_meta_bytes=229376 combine_payload_bytes=360521728 prefetch_bytes=0 reduce_bytes=0 egress_bytes=360751104 ingress_bytes=360521728 link_bytes=360751104 token_bytes=721272832 weight_movement_bytes=0 total_nvlink_bytes=721272832 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.274055032e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=393938944 dispatch_meta_bytes=229352 combine_payload_bytes=393938944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394168296 ingress_bytes=504039424 link_bytes=504039424 token_bytes=788107240 weight_movement_bytes=110100480 total_nvlink_bytes=898207720 weight_to_token_ratio=0.139702 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.766074586e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7339277704 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3670752648 ingress_bytes=5650333696 link_bytes=5650333696 total_nvlink_bytes=9321086344 weight_to_token_ratio=0.270028 prefetch_share_of_ingress=0.194857 dedup_ratio=0.101645 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048529 gemm_inner_pad_upper_bound=7239 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.826859618e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7013521496 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=3507882072 ingress_bytes=5267247104 link_bytes=5267476480 total_nvlink_bytes=8775129176 weight_to_token_ratio=0.251173 prefetch_share_of_ingress=0.209029 dedup_ratio=0.132492 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.161007950e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7411364056 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3706826968 ingress_bytes=5686345728 link_bytes=5686345728 total_nvlink_bytes=9393172696 weight_to_token_ratio=0.267401 prefetch_share_of_ingress=0.193623 dedup_ratio=0.094397 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047059 gemm_inner_pad_upper_bound=7366 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.889490464e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none + [2026-08-26 18:09:53] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 366.6 | throughput per GPU (TFLOP/s/GPU): 336.8 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.391109E+00 | loss scale: 1.0 | grad norm: 4.971 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44230 payload_rows_remote=37774 dup_rows=21306 dup_rows_remote=17662 dedup_ratio=0.325104 router_tokens_per_expert=[2090, 872, 1590, 1799, 21, 284, 274, 18, 567, 2137, 1605, 877, 7, 747, 19, 1228, 924, 633, 45, 3, 1235, 510, 64, 1190, 100, 1202, 529, 454, 106, 94, 379, 0, 209, 125, 655, 11, 339, 0, 906, 244, 112, 646, 746, 308, 0, 1076, 129, 424, 30, 94, 82, 310, 317, 856, 24, 948, 216, 475, 374, 641, 606, 149, 113, 0] payload_rows_per_dst=[7264, 6456, 6044, 4488, 6320, 5176, 4374, 4108] dup_rows_per_dst=[6632, 3644, 3164, 1240, 2932, 1706, 948, 1040] combine_rows_per_src=[7264, 6456, 6044, 4488, 6320, 5176, 4374, 4108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44096 payload_rows_remote=37364 dup_rows=21440 dup_rows_remote=16518 dedup_ratio=0.327148 router_tokens_per_expert=[1659, 563, 1640, 1429, 47, 249, 191, 49, 441, 1554, 1759, 527, 13, 411, 9, 1435, 1546, 631, 35, 2, 1001, 697, 38, 954, 63, 1144, 593, 443, 151, 266, 452, 1, 602, 323, 1100, 15, 337, 1, 427, 85, 331, 879, 678, 473, 3, 949, 65, 212, 96, 73, 295, 298, 329, 1002, 49, 834, 196, 457, 728, 763, 648, 390, 133, 4] payload_rows_per_dst=[6732, 6740, 6288, 4662, 4798, 5234, 4632, 5010] dup_rows_per_dst=[4922, 4832, 3520, 1564, 1708, 1946, 1320, 1628] combine_rows_per_src=[6732, 6740, 6288, 4662, 4798, 5234, 4632, 5010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44614 payload_rows_remote=39326 dup_rows=20922 dup_rows_remote=18256 dedup_ratio=0.319244 router_tokens_per_expert=[1757, 792, 1886, 1192, 111, 84, 429, 133, 408, 1540, 1182, 490, 0, 1079, 1, 1642, 866, 794, 9, 66, 936, 649, 171, 677, 300, 1068, 534, 436, 35, 208, 456, 1, 596, 402, 741, 11, 112, 0, 415, 167, 113, 515, 281, 641, 1, 1010, 48, 502, 28, 57, 201, 648, 523, 832, 30, 1083, 303, 234, 587, 1024, 806, 876, 49, 0] payload_rows_per_dst=[4570, 5350, 4934, 6064, 5722, 5288, 6512, 6174] dup_rows_per_dst=[912, 1890, 1670, 3526, 2246, 2666, 4064, 3948] combine_rows_per_src=[4570, 5350, 4934, 6064, 5722, 5288, 6512, 6174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44624 payload_rows_remote=38308 dup_rows=20912 dup_rows_remote=17446 dedup_ratio=0.319092 router_tokens_per_expert=[2407, 917, 2240, 1489, 211, 310, 165, 39, 227, 1793, 1498, 728, 1, 929, 9, 1242, 1303, 629, 45, 2, 735, 515, 98, 1046, 144, 1080, 445, 521, 59, 241, 286, 0, 420, 73, 414, 21, 84, 0, 578, 230, 299, 714, 377, 470, 0, 871, 137, 397, 35, 136, 490, 482, 363, 705, 31, 694, 354, 431, 495, 413, 461, 1137, 102, 0] payload_rows_per_dst=[3650, 4526, 5812, 6562, 5568, 5898, 6292, 6316] dup_rows_per_dst=[740, 1134, 2812, 3804, 1658, 3238, 4060, 3466] combine_rows_per_src=[3650, 4526, 5812, 6562, 5568, 5898, 6292, 6316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44968 payload_rows_remote=38648 dup_rows=20568 dup_rows_remote=16414 dedup_ratio=0.313843 router_tokens_per_expert=[1487, 606, 1265, 1594, 84, 324, 215, 39, 919, 2162, 1641, 946, 1, 719, 0, 1607, 836, 510, 7, 1, 1156, 528, 97, 750, 231, 924, 580, 332, 77, 85, 283, 1, 453, 153, 846, 7, 94, 8, 756, 224, 544, 578, 303, 348, 4, 666, 59, 645, 167, 207, 45, 1227, 339, 1004, 20, 963, 282, 607, 348, 473, 447, 705, 237, 2] payload_rows_per_dst=[4658, 5194, 5344, 5462, 6460, 5382, 6320, 6148] dup_rows_per_dst=[1066, 1802, 2142, 2538, 2946, 2584, 4154, 3336] combine_rows_per_src=[4658, 5194, 5344, 5462, 6460, 5382, 6320, 6148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44742 payload_rows_remote=38958 dup_rows=20794 dup_rows_remote=17964 dedup_ratio=0.317291 router_tokens_per_expert=[2192, 753, 1620, 1620, 22, 179, 207, 48, 371, 2026, 1537, 503, 1, 774, 3, 1453, 1154, 868, 52, 2, 636, 801, 15, 1405, 89, 991, 530, 569, 68, 101, 197, 1, 576, 97, 653, 6, 201, 0, 639, 111, 145, 679, 575, 582, 3, 868, 123, 362, 25, 105, 164, 822, 236, 746, 69, 1091, 362, 511, 447, 360, 672, 578, 172, 0] payload_rows_per_dst=[6694, 6106, 5784, 5058, 6090, 5352, 4728, 4930] dup_rows_per_dst=[4666, 2748, 2830, 1956, 2528, 2574, 1788, 1704] combine_rows_per_src=[6694, 6106, 5784, 5058, 6090, 5352, 4728, 4930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45334 payload_rows_remote=39568 dup_rows=20202 dup_rows_remote=18252 dedup_ratio=0.308258 router_tokens_per_expert=[1900, 659, 1386, 1303, 12, 192, 349, 86, 922, 1888, 1348, 554, 5, 685, 0, 1185, 1687, 1319, 18, 0, 860, 618, 55, 1109, 111, 915, 624, 539, 79, 209, 511, 1, 191, 178, 889, 15, 107, 0, 503, 87, 480, 685, 205, 384, 2, 1010, 258, 396, 82, 344, 88, 1006, 413, 525, 25, 953, 231, 403, 297, 465, 439, 901, 77, 0] payload_rows_per_dst=[4256, 5294, 5498, 6380, 5766, 6436, 5924, 5780] dup_rows_per_dst=[946, 1408, 2460, 3398, 1950, 3778, 3720, 2542] combine_rows_per_src=[4256, 5294, 5498, 6380, 5766, 6436, 5924, 5780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=270764032 dispatch_meta_bytes=221744 combine_payload_bytes=270764032 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=270985776 ingress_bytes=711165952 link_bytes=711165952 token_bytes=541749808 weight_movement_bytes=440401920 total_nvlink_bytes=982151728 weight_to_token_ratio=0.812925 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.236834557e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281888768 dispatch_meta_bytes=230328 combine_payload_bytes=281888768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282119096 ingress_bytes=391989248 link_bytes=391989248 token_bytes=564007864 weight_movement_bytes=110100480 total_nvlink_bytes=674108344 weight_to_token_ratio=0.195211 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.817337734e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=274591744 dispatch_meta_bytes=223016 combine_payload_bytes=274591744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=274814760 ingress_bytes=384692224 link_bytes=384692224 token_bytes=549406504 weight_movement_bytes=110100480 total_nvlink_bytes=659506984 weight_to_token_ratio=0.200399 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7296 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.690430485e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=267825152 dispatch_meta_bytes=215528 combine_payload_bytes=267825152 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=268040680 ingress_bytes=598126592 link_bytes=598126592 token_bytes=535865832 weight_movement_bytes=330301440 total_nvlink_bytes=866167272 weight_to_token_ratio=0.616388 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=10752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.040240518e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277028864 dispatch_meta_bytes=220248 combine_payload_bytes=277028864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277249112 ingress_bytes=387129344 link_bytes=387129344 token_bytes=554277976 weight_movement_bytes=110100480 total_nvlink_bytes=664378456 weight_to_token_ratio=0.198638 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.732816011e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=227688 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279478632 ingress_bytes=499451904 link_bytes=499451904 token_bytes=558729576 weight_movement_bytes=220200960 total_nvlink_bytes=778930536 weight_to_token_ratio=0.394110 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686290068e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=283623424 dispatch_meta_bytes=231280 combine_payload_bytes=283623424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=283854704 ingress_bytes=393723904 link_bytes=393723904 token_bytes=567478128 weight_movement_bytes=110100480 total_nvlink_bytes=677578608 weight_to_token_ratio=0.194017 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.847506255e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45412 payload_rows_remote=39486 dup_rows=20124 dup_rows_remote=17216 dedup_ratio=0.307068 router_tokens_per_expert=[1970, 1510, 1454, 1249, 11, 92, 143, 42, 1128, 1640, 1287, 412, 0, 683, 0, 1983, 815, 623, 31, 5, 1413, 577, 31, 937, 201, 602, 455, 711, 22, 172, 283, 1, 408, 212, 905, 22, 96, 0, 874, 265, 221, 649, 444, 884, 12, 501, 52, 561, 67, 73, 264, 598, 187, 518, 31, 1025, 266, 730, 550, 566, 550, 500, 254, 0] payload_rows_per_dst=[5754, 6104, 5198, 5926, 6062, 5592, 4688, 6088] dup_rows_per_dst=[2074, 2308, 2036, 2908, 2782, 2686, 2012, 3318] combine_rows_per_src=[5754, 6104, 5198, 5926, 6062, 5592, 4688, 6088] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54114 payload_rows_remote=47210 dup_rows=11422 dup_rows_remote=8764 dedup_ratio=0.174286 router_tokens_per_expert=[0, 0, 0, 4095, 975, 0, 3, 4092, 2597, 0, 2182, 0, 0, 0, 0, 2, 0, 2, 1, 4073, 43, 0, 0, 0, 0, 0, 14, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3032, 0, 0, 479, 0, 3973, 0, 0, 0, 0, 1, 779, 0, 6, 0, 0, 0, 2, 0, 1, 0, 2206, 0, 0, 114] payload_rows_per_dst=[8190, 6904, 8146, 8192, 6064, 7974, 4048, 4596] dup_rows_per_dst=[6878, 2658, 92, 28, 0, 930, 786, 50] combine_rows_per_src=[8190, 6904, 8146, 8192, 6064, 7974, 4048, 4596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55190 payload_rows_remote=47176 dup_rows=10346 dup_rows_remote=9522 dedup_ratio=0.157867 router_tokens_per_expert=[0, 0, 0, 4091, 712, 0, 3, 4096, 2834, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 15, 0, 0, 0, 0, 0, 4, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2714, 0, 0, 421, 0, 3998, 0, 0, 0, 0, 0, 786, 0, 2, 0, 0, 0, 1, 0, 0, 0, 2033, 0, 0, 272] payload_rows_per_dst=[1430, 5462, 7936, 8180, 7784, 8014, 8192, 8192] dup_rows_per_dst=[0, 0, 24, 8, 3312, 824, 1576, 4602] combine_rows_per_src=[1430, 5462, 7936, 8180, 7784, 8014, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55646 payload_rows_remote=47454 dup_rows=9890 dup_rows_remote=5668 dedup_ratio=0.150909 router_tokens_per_expert=[0, 0, 1, 4096, 695, 0, 4, 4096, 2616, 1, 2848, 0, 1, 1, 0, 1, 0, 0, 0, 4058, 14, 0, 0, 0, 0, 0, 29, 0, 1, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3033, 0, 0, 366, 1, 4021, 0, 0, 0, 0, 0, 671, 0, 6, 0, 0, 0, 1, 0, 0, 0, 2017, 0, 1, 92] payload_rows_per_dst=[1398, 5702, 8116, 8150, 7842, 8054, 8192, 8192] dup_rows_per_dst=[2, 2, 28, 64, 3456, 762, 1354, 4222] combine_rows_per_src=[1398, 5702, 8116, 8150, 7842, 8054, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45250 payload_rows_remote=37058 dup_rows=20286 dup_rows_remote=5280 dedup_ratio=0.309540 router_tokens_per_expert=[0, 0, 0, 4088, 589, 1, 1, 4094, 3019, 0, 2668, 0, 1, 0, 0, 0, 0, 3, 0, 4017, 37, 0, 0, 0, 0, 0, 15, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3213, 3, 0, 859, 0, 3875, 2, 4, 1, 0, 0, 781, 0, 0, 0, 0, 0, 0, 1, 11, 0, 1256, 0, 0, 134] payload_rows_per_dst=[8192, 7474, 8068, 8004, 6430, 2756, 1562, 2764] dup_rows_per_dst=[15006, 4832, 228, 30, 0, 150, 0, 40] combine_rows_per_src=[8192, 7474, 8068, 8004, 6430, 2756, 1562, 2764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54858 payload_rows_remote=46666 dup_rows=10678 dup_rows_remote=9102 dedup_ratio=0.162933 router_tokens_per_expert=[0, 0, 0, 4096, 1289, 0, 3, 4096, 2155, 0, 2156, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 91, 1, 0, 0, 0, 0, 56, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 0, 0, 660, 0, 3981, 0, 0, 0, 0, 0, 787, 0, 1, 0, 0, 0, 22, 0, 0, 0, 1975, 0, 0, 493] payload_rows_per_dst=[2582, 4312, 8184, 8190, 7206, 8000, 8192, 8192] dup_rows_per_dst=[2, 0, 184, 112, 2542, 1282, 1576, 4980] combine_rows_per_src=[2582, 4312, 8184, 8190, 7206, 8000, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56458 payload_rows_remote=48308 dup_rows=9078 dup_rows_remote=9018 dedup_ratio=0.138519 router_tokens_per_expert=[0, 0, 0, 4095, 1265, 0, 2, 4094, 3004, 0, 2040, 0, 1, 0, 1, 0, 0, 0, 0, 4075, 30, 0, 0, 0, 0, 1, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2984, 1, 0, 652, 0, 3979, 0, 0, 0, 0, 0, 1041, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1106, 0, 3, 296] payload_rows_per_dst=[8190, 7014, 8150, 8190, 5970, 7984, 8188, 2772] dup_rows_per_dst=[2534, 3078, 60, 2, 0, 1280, 2082, 42] combine_rows_per_src=[8190, 7014, 8150, 8190, 5970, 7984, 8188, 2772] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=338401280 dispatch_meta_bytes=223896 combine_payload_bytes=338401280 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=338625176 ingress_bytes=558602240 link_bytes=558602240 token_bytes=677026456 weight_movement_bytes=220200960 total_nvlink_bytes=897227416 weight_to_token_ratio=0.325247 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.715011696e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=338157568 dispatch_meta_bytes=226792 combine_payload_bytes=338157568 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=338384360 ingress_bytes=668459008 link_bytes=668459008 token_bytes=676541928 weight_movement_bytes=330301440 total_nvlink_bytes=1006843368 weight_to_token_ratio=0.488220 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.162560158e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55786 payload_rows_remote=48892 dup_rows=9750 dup_rows_remote=8524 dedup_ratio=0.148773 router_tokens_per_expert=[0, 0, 0, 4096, 773, 0, 7, 4096, 2622, 0, 2714, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 9, 1, 0, 0, 0, 0, 3, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3254, 0, 0, 315, 0, 3937, 0, 0, 0, 0, 1, 602, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1814, 0, 0, 335] payload_rows_per_dst=[1558, 6686, 8182, 8190, 6894, 7892, 8192, 8192] dup_rows_per_dst=[2, 2374, 20, 6, 1226, 612, 1210, 4300] combine_rows_per_src=[1558, 6686, 8182, 8190, 6894, 7892, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=340150272 dispatch_meta_bytes=212488 combine_payload_bytes=340150272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=340362760 ingress_bytes=450250752 link_bytes=450250752 token_bytes=680513032 weight_movement_bytes=110100480 total_nvlink_bytes=790613512 weight_to_token_ratio=0.161790 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.830601113e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265631744 dispatch_meta_bytes=169352 combine_payload_bytes=265631744 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=265801096 ingress_bytes=595933184 link_bytes=595933184 token_bytes=531432840 weight_movement_bytes=330301440 total_nvlink_bytes=861734280 weight_to_token_ratio=0.621530 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.036425821e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=283035648 dispatch_meta_bytes=226808 combine_payload_bytes=283035648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=283262456 ingress_bytes=393136128 link_bytes=393136128 token_bytes=566298104 weight_movement_bytes=110100480 total_nvlink_bytes=676398584 weight_to_token_ratio=0.194421 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.837283864e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=334501888 dispatch_meta_bytes=223072 combine_payload_bytes=334501888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=334724960 ingress_bytes=444602368 link_bytes=444602368 token_bytes=669226848 weight_movement_bytes=110100480 total_nvlink_bytes=779327328 weight_to_token_ratio=0.164519 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=26240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.732366425e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=346271744 dispatch_meta_bytes=229304 combine_payload_bytes=346271744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=346501048 ingress_bytes=456372224 link_bytes=456372224 token_bytes=692772792 weight_movement_bytes=110100480 total_nvlink_bytes=802873272 weight_to_token_ratio=0.158927 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.937063579e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=350457856 dispatch_meta_bytes=229664 combine_payload_bytes=350457856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=350687520 ingress_bytes=460558336 link_bytes=460558336 token_bytes=701145376 weight_movement_bytes=110100480 total_nvlink_bytes=811245856 weight_to_token_ratio=0.157029 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=24448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.009866951e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65248 payload_rows_remote=57056 dup_rows=288 dup_rows_remote=288 dedup_ratio=0.004395 router_tokens_per_expert=[0, 0, 0, 4096, 3, 3960, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086] payload_rows_per_dst=[7924, 8192, 8192, 8192, 8192, 8192, 8192, 8172] dup_rows_per_dst=[2, 0, 254, 32, 0, 0, 0, 0] combine_rows_per_src=[7924, 8192, 8192, 8192, 8192, 8192, 8192, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65208 payload_rows_remote=57024 dup_rows=328 dup_rows_remote=328 dedup_ratio=0.005005 router_tokens_per_expert=[0, 0, 0, 4096, 1, 4020, 4095, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 106, 4096, 0, 9, 3, 0, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4013] payload_rows_per_dst=[8042, 8184, 8190, 8192, 8190, 8192, 8192, 8026] dup_rows_per_dst=[0, 0, 88, 236, 4, 0, 0, 0] combine_rows_per_src=[8042, 8184, 8190, 8192, 8190, 8192, 8192, 8026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65090 payload_rows_remote=56946 dup_rows=446 dup_rows_remote=128 dedup_ratio=0.006805 router_tokens_per_expert=[0, 0, 0, 4096, 3, 4059, 4095, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 34, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 4070] payload_rows_per_dst=[8122, 8188, 8192, 7886, 8192, 8174, 8192, 8144] dup_rows_per_dst=[0, 0, 46, 72, 6, 2, 2, 318] combine_rows_per_src=[8122, 8188, 8192, 7886, 8192, 8174, 8192, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65358 payload_rows_remote=57166 dup_rows=178 dup_rows_remote=178 dedup_ratio=0.002716 router_tokens_per_expert=[0, 0, 0, 4096, 0, 4007, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 17, 4096, 0, 2, 1, 0, 0, 8, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8014, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 124, 40, 14, 0, 0, 0] combine_rows_per_src=[8014, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64228 payload_rows_remote=56068 dup_rows=1308 dup_rows_remote=110 dedup_ratio=0.019958 router_tokens_per_expert=[0, 0, 0, 4095, 14, 4068, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 30, 4096, 0, 1, 5, 0, 1, 0, 0, 0, 0, 0, 4091, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 4061] payload_rows_per_dst=[8160, 8186, 7388, 7830, 8162, 8188, 8190, 8124] dup_rows_per_dst=[1198, 20, 32, 52, 2, 4, 0, 0] combine_rows_per_src=[8160, 8186, 7388, 7830, 8162, 8188, 8190, 8124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65476 payload_rows_remote=57284 dup_rows=60 dup_rows_remote=30 dedup_ratio=0.000916 router_tokens_per_expert=[0, 0, 0, 4096, 4, 4072, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 0, 10, 4095, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4089] payload_rows_per_dst=[8152, 8190, 8192, 8190, 8192, 8190, 8192, 8178] dup_rows_per_dst=[0, 0, 30, 26, 2, 2, 0, 0] combine_rows_per_src=[8152, 8190, 8192, 8190, 8192, 8190, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57714 payload_rows_remote=49526 dup_rows=7822 dup_rows_remote=7820 dedup_ratio=0.119354 router_tokens_per_expert=[0, 0, 0, 4096, 998, 0, 31, 4096, 2562, 0, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 14, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3405, 0, 0, 549, 0, 3962, 0, 0, 0, 0, 0, 1088, 0, 9, 0, 0, 0, 0, 0, 2, 0, 1450, 0, 0, 409] payload_rows_per_dst=[7660, 6798, 8102, 8188, 6810, 7938, 8192, 4026] dup_rows_per_dst=[1912, 2240, 16, 2, 0, 1084, 2194, 374] combine_rows_per_src=[7660, 6798, 8102, 8188, 6810, 7938, 8192, 4026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65208 payload_rows_remote=57016 dup_rows=328 dup_rows_remote=328 dedup_ratio=0.005005 router_tokens_per_expert=[0, 0, 0, 4096, 0, 3933, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 101, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095] payload_rows_per_dst=[7866, 8192, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 110, 202, 0, 16, 0, 0] combine_rows_per_src=[7866, 8192, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=408977408 dispatch_meta_bytes=229376 combine_payload_bytes=408977408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409206784 ingress_bytes=519077888 link_bytes=519077888 token_bytes=818184192 weight_movement_bytes=110100480 total_nvlink_bytes=928284672 weight_to_token_ratio=0.134567 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.027618209e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=408188928 dispatch_meta_bytes=228296 combine_payload_bytes=408188928 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=408417224 ingress_bytes=628389888 link_bytes=628389888 token_bytes=816606152 weight_movement_bytes=220200960 total_nvlink_bytes=1036807112 weight_to_token_ratio=0.269654 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.092873368e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=408748032 dispatch_meta_bytes=229408 combine_payload_bytes=408748032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=408977440 ingress_bytes=518848512 link_bytes=518848512 token_bytes=817725472 weight_movement_bytes=110100480 total_nvlink_bytes=927825952 weight_to_token_ratio=0.134642 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.023628983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=409765888 dispatch_meta_bytes=229376 combine_payload_bytes=409765888 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=409995264 ingress_bytes=629966848 link_bytes=629966848 token_bytes=819761152 weight_movement_bytes=220200960 total_nvlink_bytes=1039962112 weight_to_token_ratio=0.268616 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.095615960e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=410611712 dispatch_meta_bytes=229256 combine_payload_bytes=410611712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410840968 ingress_bytes=520712192 link_bytes=520712192 token_bytes=821452680 weight_movement_bytes=110100480 total_nvlink_bytes=931553160 weight_to_token_ratio=0.134031 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.056041443e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=401895424 dispatch_meta_bytes=224712 combine_payload_bytes=401895424 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=402120136 ingress_bytes=842297344 link_bytes=842297344 token_bytes=804015560 weight_movement_bytes=440401920 total_nvlink_bytes=1244417480 weight_to_token_ratio=0.547753 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.464893615e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=355002368 dispatch_meta_bytes=229384 combine_payload_bytes=355002368 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=355231752 ingress_bytes=575203328 link_bytes=575203328 token_bytes=710234120 weight_movement_bytes=220200960 total_nvlink_bytes=930435080 weight_to_token_ratio=0.310040 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.000373192e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=408690688 dispatch_meta_bytes=229376 combine_payload_bytes=408690688 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=408920064 ingress_bytes=628891648 link_bytes=628891648 token_bytes=817610752 weight_movement_bytes=220200960 total_nvlink_bytes=1037811712 weight_to_token_ratio=0.269322 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.093746011e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62210 payload_rows_remote=54128 dup_rows=3326 dup_rows_remote=3326 dedup_ratio=0.050751 router_tokens_per_expert=[0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 2504, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 4096, 0, 0, 1460, 0, 3, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 35, 0, 4096, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8188, 5012, 8192, 8192, 8192, 8082, 8160, 8192] dup_rows_per_dst=[0, 0, 332, 2924, 70, 0, 0, 0] combine_rows_per_src=[8188, 5012, 8192, 8192, 8192, 8082, 8160, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58038 payload_rows_remote=49848 dup_rows=7498 dup_rows_remote=7494 dedup_ratio=0.114410 router_tokens_per_expert=[0, 0, 2, 0, 0, 4096, 0, 0, 4, 0, 0, 2833, 0, 0, 0, 0, 0, 0, 269, 0, 0, 0, 4096, 0, 0, 1038, 0, 2, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 1, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8126, 5674, 3388, 8192, 8192, 8088, 8188, 8190] dup_rows_per_dst=[4, 0, 252, 6638, 40, 498, 62, 4] combine_rows_per_src=[8126, 5674, 3388, 8192, 8192, 8088, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59052 payload_rows_remote=50860 dup_rows=6484 dup_rows_remote=766 dedup_ratio=0.098938 router_tokens_per_expert=[0, 0, 13, 0, 0, 4096, 0, 0, 18, 0, 0, 2971, 0, 0, 0, 0, 0, 0, 362, 0, 0, 0, 4096, 0, 0, 871, 0, 16, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 3, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 4093, 0] payload_rows_per_dst=[8192, 8192, 8192, 1774, 8192, 8144, 8178, 8188] dup_rows_per_dst=[26, 5718, 724, 0, 16, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 1774, 8192, 8144, 8178, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63092 payload_rows_remote=54904 dup_rows=2444 dup_rows_remote=2444 dedup_ratio=0.037292 router_tokens_per_expert=[0, 0, 3, 0, 0, 4096, 0, 0, 13, 0, 0, 2884, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 4096, 0, 0, 685, 0, 3, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 5794, 8192, 8192, 8192, 8150, 8188, 8192] dup_rows_per_dst=[6, 0, 1052, 1360, 26, 0, 0, 0] combine_rows_per_src=[8192, 5794, 8192, 8192, 8192, 8150, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=387989504 dispatch_meta_bytes=229816 combine_payload_bytes=387989504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=388219320 ingress_bytes=498089984 link_bytes=498089984 token_bytes=776208824 weight_movement_bytes=110100480 total_nvlink_bytes=886309304 weight_to_token_ratio=0.141844 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662604039e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59336 payload_rows_remote=51144 dup_rows=6200 dup_rows_remote=6186 dedup_ratio=0.094604 router_tokens_per_expert=[0, 0, 6, 0, 0, 4096, 0, 0, 6, 0, 0, 2550, 0, 0, 0, 0, 0, 0, 427, 0, 0, 0, 4095, 0, 0, 1104, 0, 37, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 6, 0, 4091, 1, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8192, 8192, 8190, 2280, 7946, 8156, 8188, 8192] dup_rows_per_dst=[14, 5082, 1098, 2, 2, 0, 0, 2] combine_rows_per_src=[8192, 8192, 8190, 2280, 7946, 8156, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61050 payload_rows_remote=52858 dup_rows=4486 dup_rows_remote=4020 dedup_ratio=0.068451 router_tokens_per_expert=[0, 0, 3, 0, 0, 4096, 0, 0, 19, 0, 0, 3161, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 4096, 0, 0, 740, 0, 0, 0, 1, 4036, 0, 0, 0, 0, 0, 0, 28, 0, 4095, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 7204, 8192, 4752, 8190, 8138, 8190, 8192] dup_rows_per_dst=[6, 3200, 466, 758, 56, 0, 0, 0] combine_rows_per_src=[8192, 7204, 8192, 4752, 8190, 8138, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=357310464 dispatch_meta_bytes=229368 combine_payload_bytes=357310464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=357539832 ingress_bytes=467410944 link_bytes=467410944 token_bytes=714850296 weight_movement_bytes=110100480 total_nvlink_bytes=824950776 weight_to_token_ratio=0.154019 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.129045075e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=364564480 dispatch_meta_bytes=206504 combine_payload_bytes=364564480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364770984 ingress_bytes=474664960 link_bytes=474664960 token_bytes=729335464 weight_movement_bytes=110100480 total_nvlink_bytes=839435944 weight_to_token_ratio=0.150960 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.255204344e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=393551872 dispatch_meta_bytes=229392 combine_payload_bytes=393551872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=393781264 ingress_bytes=503652352 link_bytes=503652352 token_bytes=787333136 weight_movement_bytes=110100480 total_nvlink_bytes=897433616 weight_to_token_ratio=0.139840 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.759342767e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63308 payload_rows_remote=55116 dup_rows=2228 dup_rows_remote=2190 dedup_ratio=0.033997 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 2991, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 4096, 0, 0, 1029, 0, 1, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 19, 0, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 5990, 8192, 8192, 8192, 8166, 8192, 8192] dup_rows_per_dst=[0, 0, 284, 1906, 38, 0, 0, 0] combine_rows_per_src=[8192, 5990, 8192, 8192, 8192, 8166, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65400 payload_rows_remote=57208 dup_rows=136 dup_rows_remote=126 dedup_ratio=0.002075 router_tokens_per_expert=[0, 0, 0, 4096, 1, 4027, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 3, 4096, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8056, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 122, 10, 4, 0, 0, 0] combine_rows_per_src=[8056, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=366600192 dispatch_meta_bytes=229320 combine_payload_bytes=366600192 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=366829512 ingress_bytes=696901632 link_bytes=696901632 token_bytes=733429704 weight_movement_bytes=330301440 total_nvlink_bytes=1063731144 weight_to_token_ratio=0.450352 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.212026559e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=378886144 dispatch_meta_bytes=227512 combine_payload_bytes=378886144 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=379113656 ingress_bytes=709187584 link_bytes=709187584 token_bytes=757999800 weight_movement_bytes=330301440 total_nvlink_bytes=1088301240 weight_to_token_ratio=0.435754 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.233393850e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=395071488 dispatch_meta_bytes=229224 combine_payload_bytes=395071488 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=395300712 ingress_bytes=615272448 link_bytes=615272448 token_bytes=790372200 weight_movement_bytes=220200960 total_nvlink_bytes=1010573160 weight_to_token_ratio=0.278604 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.070059982e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54330 payload_rows_remote=46138 dup_rows=11206 dup_rows_remote=10840 dedup_ratio=0.170990 router_tokens_per_expert=[0, 1780, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 0, 0, 2104, 0, 29, 0, 0, 0, 0, 0, 0, 946, 0, 4096, 0, 0, 0, 0, 4096, 0, 183, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 3548, 2089, 0, 4096, 0, 0, 0, 1339, 0] payload_rows_per_dst=[3560, 8170, 8192, 8140, 4450, 8192, 7580, 6046] dup_rows_per_dst=[0, 0, 622, 4258, 386, 366, 4764, 810] combine_rows_per_src=[3560, 8170, 8192, 8140, 4450, 8192, 7580, 6046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=410066944 dispatch_meta_bytes=229336 combine_payload_bytes=410066944 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=410296280 ingress_bytes=740368384 link_bytes=740368384 token_bytes=820363224 weight_movement_bytes=330301440 total_nvlink_bytes=1150664664 weight_to_token_ratio=0.402628 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.287622389e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57062 payload_rows_remote=48870 dup_rows=8474 dup_rows_remote=8474 dedup_ratio=0.129303 router_tokens_per_expert=[0, 2113, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 0, 0, 2252, 0, 14, 1, 0, 0, 0, 0, 0, 1466, 0, 4095, 0, 0, 0, 0, 4096, 0, 83, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 3246, 1810, 0, 4096, 0, 0, 0, 981, 0] payload_rows_per_dst=[8170, 8192, 8192, 4532, 8190, 8192, 6510, 5084] dup_rows_per_dst=[4208, 0, 666, 2, 2932, 168, 0, 498] combine_rows_per_src=[8170, 8192, 8192, 4532, 8190, 8192, 6510, 5084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47524 payload_rows_remote=39332 dup_rows=18012 dup_rows_remote=4576 dedup_ratio=0.274841 router_tokens_per_expert=[0, 1496, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 0, 0, 0, 2245, 0, 25, 0, 0, 0, 0, 0, 0, 1945, 0, 4094, 0, 0, 0, 0, 4096, 0, 29, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 3559, 1542, 0, 4096, 0, 0, 0, 1082, 0] payload_rows_per_dst=[2992, 8192, 802, 8140, 3890, 8192, 7124, 8192] dup_rows_per_dst=[0, 0, 0, 4512, 0, 64, 0, 13436] combine_rows_per_src=[2992, 8192, 802, 8140, 3890, 8192, 7124, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52262 payload_rows_remote=44378 dup_rows=13274 dup_rows_remote=8762 dedup_ratio=0.202545 router_tokens_per_expert=[0, 1218, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 830, 0, 0, 0, 0, 1542, 0, 16, 0, 0, 0, 0, 0, 0, 1974, 0, 4096, 0, 0, 0, 0, 4096, 0, 122, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3556, 2145, 0, 4096, 0, 0, 0, 965, 0] payload_rows_per_dst=[2436, 8190, 7102, 8082, 3948, 8192, 7884, 6428] dup_rows_per_dst=[0, 0, 1518, 3068, 0, 244, 4512, 3932] combine_rows_per_src=[2436, 8190, 7102, 8082, 3948, 8192, 7884, 6428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=330717184 dispatch_meta_bytes=227912 combine_payload_bytes=330717184 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=330945096 ingress_bytes=440817664 link_bytes=440817664 token_bytes=661662280 weight_movement_bytes=110100480 total_nvlink_bytes=771762760 weight_to_token_ratio=0.166400 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.666544197e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54726 payload_rows_remote=46544 dup_rows=10810 dup_rows_remote=3908 dedup_ratio=0.164948 router_tokens_per_expert=[0, 2350, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 678, 0, 0, 0, 0, 1812, 0, 9, 0, 0, 0, 0, 0, 0, 1102, 0, 4094, 0, 0, 0, 0, 4096, 0, 45, 0, 0, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3385, 1770, 0, 4096, 0, 0, 0, 1187, 0] payload_rows_per_dst=[8182, 8190, 8192, 3634, 7300, 7110, 6742, 5376] dup_rows_per_dst=[6902, 28, 1356, 8, 1938, 40, 0, 538] combine_rows_per_src=[8182, 8190, 8192, 3634, 7300, 7110, 6742, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58110 payload_rows_remote=49918 dup_rows=7426 dup_rows_remote=6698 dedup_ratio=0.113312 router_tokens_per_expert=[0, 2052, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 2396, 0, 35, 1, 0, 0, 0, 0, 0, 1017, 0, 4095, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 3494, 1587, 0, 4096, 0, 0, 0, 1373, 0] payload_rows_per_dst=[8182, 8192, 8192, 4830, 8190, 8192, 6994, 5338] dup_rows_per_dst=[4000, 0, 728, 34, 2034, 48, 0, 582] combine_rows_per_src=[8182, 8192, 8192, 4830, 8190, 8192, 6994, 5338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=350300160 dispatch_meta_bytes=229376 combine_payload_bytes=350300160 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=350529536 ingress_bytes=570501120 link_bytes=570501120 token_bytes=700829696 weight_movement_bytes=220200960 total_nvlink_bytes=921030656 weight_to_token_ratio=0.314200 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.921952790e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281931776 dispatch_meta_bytes=175632 combine_payload_bytes=281931776 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=282107408 ingress_bytes=502132736 link_bytes=502132736 token_bytes=564039184 weight_movement_bytes=220200960 total_nvlink_bytes=784240144 weight_to_token_ratio=0.390400 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.732914145e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=318101504 dispatch_meta_bytes=212560 combine_payload_bytes=318101504 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=318314064 ingress_bytes=648402944 link_bytes=648402944 token_bytes=636415568 weight_movement_bytes=330301440 total_nvlink_bytes=966717008 weight_to_token_ratio=0.519003 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.127679364e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56206 payload_rows_remote=48014 dup_rows=9330 dup_rows_remote=6504 dedup_ratio=0.142365 router_tokens_per_expert=[0, 2398, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 0, 0, 0, 0, 2088, 0, 38, 0, 0, 0, 0, 0, 0, 1413, 0, 4096, 0, 0, 0, 0, 4096, 0, 37, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3610, 1040, 0, 4096, 0, 0, 0, 670, 0] payload_rows_per_dst=[4910, 8190, 8192, 8022, 8192, 8192, 7220, 3288] dup_rows_per_dst=[122, 0, 2050, 4126, 2826, 74, 0, 132] combine_rows_per_src=[4910, 8190, 8192, 8022, 8192, 8192, 7220, 3288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=333627392 dispatch_meta_bytes=201808 combine_payload_bytes=333627392 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=333829200 ingress_bytes=553828352 link_bytes=553828352 token_bytes=667456592 weight_movement_bytes=220200960 total_nvlink_bytes=887657552 weight_to_token_ratio=0.329911 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=16384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.631985932e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=357812224 dispatch_meta_bytes=226464 combine_payload_bytes=357812224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=358038688 ingress_bytes=467912704 link_bytes=467912704 token_bytes=715850912 weight_movement_bytes=110100480 total_nvlink_bytes=825951392 weight_to_token_ratio=0.153804 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=28160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.137771507e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62742 payload_rows_remote=54550 dup_rows=2794 dup_rows_remote=612 dedup_ratio=0.042633 router_tokens_per_expert=[0, 0, 7, 0, 0, 4096, 0, 0, 8, 0, 0, 2731, 0, 0, 0, 0, 0, 0, 299, 0, 0, 0, 4096, 0, 0, 1133, 0, 10, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 5478, 8192, 8192, 8192, 8114, 8190, 8192] dup_rows_per_dst=[14, 0, 598, 2182, 0, 0, 0, 0] combine_rows_per_src=[8192, 5478, 8192, 8192, 8192, 8114, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=344164352 dispatch_meta_bytes=218072 combine_payload_bytes=344164352 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=344382424 ingress_bytes=784566272 link_bytes=784566272 token_bytes=688546776 weight_movement_bytes=440401920 total_nvlink_bytes=1128948696 weight_to_token_ratio=0.639611 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.364489786e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57466 payload_rows_remote=49274 dup_rows=8070 dup_rows_remote=7976 dedup_ratio=0.123138 router_tokens_per_expert=[0, 0, 0, 4096, 0, 162, 0, 0, 0, 0, 0, 0, 0, 2071, 0, 135, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 4096, 0, 3962, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 4412, 8192, 4054, 8192, 8192, 8040, 8192] dup_rows_per_dst=[324, 0, 0, 968, 0, 94, 6684, 0] combine_rows_per_src=[8192, 4412, 8192, 4054, 8192, 8192, 8040, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=391014400 dispatch_meta_bytes=220648 combine_payload_bytes=391014400 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=391235048 ingress_bytes=611215360 link_bytes=611215360 token_bytes=782249448 weight_movement_bytes=220200960 total_nvlink_bytes=1002450408 weight_to_token_ratio=0.281497 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=22400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.063004039e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57216 payload_rows_remote=49182 dup_rows=8320 dup_rows_remote=3772 dedup_ratio=0.126953 router_tokens_per_expert=[0, 0, 0, 4096, 0, 185, 0, 0, 0, 0, 0, 0, 0, 2340, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1393, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 2, 301, 0, 0, 4096, 0, 3938, 0, 4095, 0, 0, 16, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8192, 8034, 8192, 8192, 8192, 8192, 32, 8190] dup_rows_per_dst=[370, 4548, 0, 2784, 12, 606, 0, 0] combine_rows_per_src=[8192, 8034, 8192, 8192, 8192, 8192, 32, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50996 payload_rows_remote=42806 dup_rows=14540 dup_rows_remote=14534 dedup_ratio=0.221863 router_tokens_per_expert=[0, 0, 0, 4096, 0, 82, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 2, 49, 0, 0, 4096, 0, 4002, 0, 4095, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[6534, 4974, 8192, 3134, 8186, 3596, 8190, 8190] dup_rows_per_dst=[138, 0, 0, 0, 4, 72, 14320, 6] combine_rows_per_src=[6534, 4974, 8192, 3134, 8186, 3596, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57000 payload_rows_remote=48808 dup_rows=8536 dup_rows_remote=398 dedup_ratio=0.130249 router_tokens_per_expert=[0, 0, 0, 4096, 0, 95, 0, 0, 0, 0, 0, 0, 0, 2814, 0, 70, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 4096, 0, 4068, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 5768, 8192, 2080, 8192, 8192, 8192, 8192] dup_rows_per_dst=[190, 0, 0, 0, 0, 208, 8138, 0] combine_rows_per_src=[8192, 5768, 8192, 2080, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=353196032 dispatch_meta_bytes=229000 combine_payload_bytes=353196032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=353425032 ingress_bytes=463296512 link_bytes=463296512 token_bytes=706621064 weight_movement_bytes=110100480 total_nvlink_bytes=816721544 weight_to_token_ratio=0.155813 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.057488336e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57130 payload_rows_remote=48938 dup_rows=8406 dup_rows_remote=8000 dedup_ratio=0.128265 router_tokens_per_expert=[0, 0, 0, 4096, 0, 183, 0, 0, 0, 0, 0, 0, 0, 2076, 0, 61, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1649, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 4096, 0, 3932, 0, 4094, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8192, 7984, 8192, 8192, 8178, 8192, 12, 8188] dup_rows_per_dst=[406, 4154, 0, 3294, 2, 550, 0, 0] combine_rows_per_src=[8192, 7984, 8192, 8192, 8178, 8192, 12, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57306 payload_rows_remote=49114 dup_rows=8230 dup_rows_remote=8230 dedup_ratio=0.125580 router_tokens_per_expert=[0, 0, 0, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0, 2626, 0, 39, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 961, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4, 472, 0, 0, 4096, 0, 4057, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8154, 8192, 8192, 8192, 8192, 0, 8192] dup_rows_per_dst=[66, 5290, 0, 1920, 2, 952, 0, 0] combine_rows_per_src=[8192, 8154, 8192, 8192, 8192, 8192, 0, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=352536576 dispatch_meta_bytes=211816 combine_payload_bytes=352536576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352748392 ingress_bytes=462637056 link_bytes=462637056 token_bytes=705284968 weight_movement_bytes=110100480 total_nvlink_bytes=815385448 weight_to_token_ratio=0.156108 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.046019311e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61190 payload_rows_remote=52998 dup_rows=4346 dup_rows_remote=4346 dedup_ratio=0.066315 router_tokens_per_expert=[0, 0, 0, 4096, 0, 46, 0, 0, 0, 0, 0, 0, 0, 1894, 0, 70, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1519, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 0, 4096, 0, 4055, 0, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 3928, 8192, 8192, 8192, 8192, 8110, 8192] dup_rows_per_dst=[92, 0, 0, 3038, 0, 1190, 26, 0] combine_rows_per_src=[8192, 3928, 8192, 8192, 8192, 8192, 8110, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=306833408 dispatch_meta_bytes=229360 combine_payload_bytes=306833408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=307062768 ingress_bytes=416933888 link_bytes=416933888 token_bytes=613896176 weight_movement_bytes=110100480 total_nvlink_bytes=723996656 weight_to_token_ratio=0.179347 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.251166051e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=349855744 dispatch_meta_bytes=196824 combine_payload_bytes=349855744 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=350052568 ingress_bytes=790257664 link_bytes=790257664 token_bytes=699908312 weight_movement_bytes=440401920 total_nvlink_bytes=1140310232 weight_to_token_ratio=0.629228 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=18048 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.374388053e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=350787584 dispatch_meta_bytes=227752 combine_payload_bytes=350787584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=351015336 ingress_bytes=570988544 link_bytes=570988544 token_bytes=701802920 weight_movement_bytes=220200960 total_nvlink_bytes=922003880 weight_to_token_ratio=0.313765 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.930429895e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=352049152 dispatch_meta_bytes=229376 combine_payload_bytes=352049152 prefetch_bytes=0 reduce_bytes=0 egress_bytes=352278528 ingress_bytes=352049152 link_bytes=352278528 token_bytes=704327680 weight_movement_bytes=0 total_nvlink_bytes=704327680 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.126703001e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56730 payload_rows_remote=52590 dup_rows=8806 dup_rows_remote=8798 dedup_ratio=0.134369 router_tokens_per_expert=[0, 1848, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 770, 0, 0, 0, 0, 2062, 0, 12, 0, 0, 0, 0, 0, 0, 1534, 0, 4096, 0, 0, 0, 0, 4096, 0, 39, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 3100, 2175, 0, 4096, 0, 0, 0, 753, 0] payload_rows_per_dst=[8192, 8192, 8192, 4140, 8192, 8192, 6206, 5424] dup_rows_per_dst=[3680, 0, 1540, 8, 3068, 78, 0, 432] combine_rows_per_src=[8192, 8192, 8192, 4140, 8192, 8192, 6206, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=379889664 dispatch_meta_bytes=229376 combine_payload_bytes=379889664 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=380119040 ingress_bytes=600090624 link_bytes=600090624 token_bytes=760008704 weight_movement_bytes=220200960 total_nvlink_bytes=980209664 weight_to_token_ratio=0.289735 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.043656293e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63406 payload_rows_remote=55214 dup_rows=2130 dup_rows_remote=2130 dedup_ratio=0.032501 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3149, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 1065, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6298, 8192, 8192, 7956, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2130, 0] combine_rows_per_src=[8192, 6298, 8192, 8192, 7956, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60258 payload_rows_remote=52658 dup_rows=5278 dup_rows_remote=1050 dedup_ratio=0.080536 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 4096, 3161, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 964, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7600, 8190, 8192, 8138, 8184, 3570, 8192] dup_rows_per_dst=[2, 4228, 8, 0, 516, 30, 492, 2] combine_rows_per_src=[8192, 7600, 8190, 8192, 8138, 8184, 3570, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63626 payload_rows_remote=55434 dup_rows=1910 dup_rows_remote=1910 dedup_ratio=0.029144 router_tokens_per_expert=[0, 2, 0, 0, 0, 2, 0, 0, 4096, 3160, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 936, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8158, 6328, 8192, 8192, 8184, 8188, 8192, 8192] dup_rows_per_dst=[8, 30, 0, 0, 0, 0, 1872, 0] combine_rows_per_src=[8158, 6328, 8192, 8192, 8184, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=376965120 dispatch_meta_bytes=245552 combine_payload_bytes=376965120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=377210672 ingress_bytes=487065600 link_bytes=487065600 token_bytes=754175792 weight_movement_bytes=110100480 total_nvlink_bytes=864276272 weight_to_token_ratio=0.145988 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.470871869e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63670 payload_rows_remote=55478 dup_rows=1866 dup_rows_remote=2 dedup_ratio=0.028473 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 4096, 3236, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 932, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6472, 8192, 8192, 8046, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 1864, 0] combine_rows_per_src=[8192, 6472, 8192, 8192, 8046, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=395773952 dispatch_meta_bytes=229376 combine_payload_bytes=395773952 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=396003328 ingress_bytes=615974912 link_bytes=615974912 token_bytes=791777280 weight_movement_bytes=220200960 total_nvlink_bytes=1011978240 weight_to_token_ratio=0.278110 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.071281682e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58730 payload_rows_remote=50538 dup_rows=6806 dup_rows_remote=6796 dedup_ratio=0.103851 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 0, 0, 4096, 3391, 0, 7, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 1, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 723, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 8150, 8180, 1446, 8190] dup_rows_per_dst=[10, 6796, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8188, 8192, 8150, 8180, 1446, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64292 payload_rows_remote=56102 dup_rows=1244 dup_rows_remote=1244 dedup_ratio=0.018982 router_tokens_per_expert=[0, 2, 0, 0, 0, 1, 0, 0, 4096, 3486, 0, 3, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 616, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6972, 8190, 8192, 8176, 8186, 8192, 8192] dup_rows_per_dst=[6, 6, 0, 0, 0, 0, 1232, 0] combine_rows_per_src=[8192, 6972, 8190, 8192, 8176, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64614 payload_rows_remote=56422 dup_rows=922 dup_rows_remote=922 dedup_ratio=0.014069 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 4096, 3635, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 460, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7270, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 920, 0] combine_rows_per_src=[8192, 7270, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=377452544 dispatch_meta_bytes=214832 combine_payload_bytes=377452544 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=377667376 ingress_bytes=597653504 link_bytes=597653504 token_bytes=755119920 weight_movement_bytes=220200960 total_nvlink_bytes=975320880 weight_to_token_ratio=0.291611 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=25984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.039417741e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=397350912 dispatch_meta_bytes=229376 combine_payload_bytes=397350912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=397580288 ingress_bytes=507451392 link_bytes=507451392 token_bytes=794931200 weight_movement_bytes=110100480 total_nvlink_bytes=905031680 weight_to_token_ratio=0.138503 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.825414321e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=397666304 dispatch_meta_bytes=221920 combine_payload_bytes=397666304 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=397888224 ingress_bytes=948168704 link_bytes=948168704 token_bytes=795554528 weight_movement_bytes=550502400 total_nvlink_bytes=1346056928 weight_to_token_ratio=0.691973 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=25728 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.649021323e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=362256384 dispatch_meta_bytes=229336 combine_payload_bytes=362256384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362485720 ingress_bytes=472356864 link_bytes=472356864 token_bytes=724742104 weight_movement_bytes=110100480 total_nvlink_bytes=834842584 weight_to_token_ratio=0.151917 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.215062759e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=402139136 dispatch_meta_bytes=229384 combine_payload_bytes=402139136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402368520 ingress_bytes=512239616 link_bytes=512239616 token_bytes=804507656 weight_movement_bytes=110100480 total_nvlink_bytes=914608136 weight_to_token_ratio=0.136854 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.908689412e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=404432896 dispatch_meta_bytes=229376 combine_payload_bytes=404432896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404662272 ingress_bytes=514533376 link_bytes=514533376 token_bytes=809095168 weight_movement_bytes=110100480 total_nvlink_bytes=919195648 weight_to_token_ratio=0.136079 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.948581671e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59384 payload_rows_remote=51192 dup_rows=6152 dup_rows_remote=2848 dedup_ratio=0.093872 router_tokens_per_expert=[0, 0, 0, 4096, 0, 36, 0, 0, 0, 0, 0, 0, 0, 1933, 0, 43, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 4096, 0, 4065, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0] payload_rows_per_dst=[8192, 6440, 8192, 8192, 8192, 8192, 3792, 8192] dup_rows_per_dst=[72, 1850, 0, 3304, 0, 924, 0, 2] combine_rows_per_src=[8192, 6440, 8192, 8192, 8192, 8192, 3792, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61228 payload_rows_remote=53036 dup_rows=4308 dup_rows_remote=4308 dedup_ratio=0.065735 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1535, 0, 0, 0, 0, 0, 525, 4096, 0, 0, 0, 18, 4096, 0, 0, 3994, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2307, 0, 0, 3909, 0, 0] payload_rows_per_dst=[4842, 8084, 8004, 8192, 8192, 8192, 8192, 7530] dup_rows_per_dst=[476, 1030, 148, 0, 0, 0, 0, 2654] combine_rows_per_src=[4842, 8084, 8004, 8192, 8192, 8192, 8192, 7530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54028 payload_rows_remote=45836 dup_rows=11508 dup_rows_remote=2 dedup_ratio=0.175598 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1222, 0, 0, 1, 0, 0, 1280, 4096, 0, 0, 0, 0, 4096, 0, 0, 4032, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1823, 0, 0, 3930, 0, 0] payload_rows_per_dst=[2444, 2560, 8064, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 11506] combine_rows_per_src=[2444, 2560, 8064, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59990 payload_rows_remote=51798 dup_rows=5546 dup_rows_remote=2460 dedup_ratio=0.084625 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1278, 0, 0, 2, 0, 0, 1541, 4096, 0, 0, 0, 2, 4096, 0, 0, 4011, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1493, 0, 0, 3961, 0, 0] payload_rows_per_dst=[8022, 8192, 8022, 8192, 8192, 8192, 8192, 2986] dup_rows_per_dst=[2456, 3086, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[8022, 8192, 8022, 8192, 8192, 8192, 8192, 2986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=366944256 dispatch_meta_bytes=216160 combine_payload_bytes=366944256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=367160416 ingress_bytes=477044736 link_bytes=477044736 token_bytes=734104672 weight_movement_bytes=110100480 total_nvlink_bytes=844205152 weight_to_token_ratio=0.149979 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.296592563e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54366 payload_rows_remote=46174 dup_rows=11170 dup_rows_remote=11170 dedup_ratio=0.170441 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1988, 0, 0, 0, 0, 0, 696, 4096, 0, 0, 0, 7, 4096, 0, 0, 4018, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1588, 0, 0, 3988, 0, 0] payload_rows_per_dst=[3976, 1392, 8038, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 806, 6, 0, 0, 0, 10358] combine_rows_per_src=[3976, 1392, 8038, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61212 payload_rows_remote=53242 dup_rows=4324 dup_rows_remote=1398 dedup_ratio=0.065979 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1533, 0, 0, 1, 0, 0, 688, 4096, 0, 0, 0, 1, 4096, 0, 0, 4048, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2102, 0, 0, 3915, 0, 0] payload_rows_per_dst=[7970, 8192, 8096, 8174, 8192, 8192, 8192, 4204] dup_rows_per_dst=[2926, 1396, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[7970, 8192, 8096, 8174, 8192, 8192, 8192, 4204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=380162048 dispatch_meta_bytes=229376 combine_payload_bytes=380162048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380391424 ingress_bytes=490262528 link_bytes=490262528 token_bytes=760553472 weight_movement_bytes=110100480 total_nvlink_bytes=870653952 weight_to_token_ratio=0.144764 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.526471705e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60060 payload_rows_remote=51868 dup_rows=5476 dup_rows_remote=5476 dedup_ratio=0.083557 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 0, 0, 0, 1727, 4096, 0, 0, 0, 0, 4096, 0, 0, 4077, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1493, 0, 0, 3941, 0, 0] payload_rows_per_dst=[7960, 8192, 8154, 8192, 8192, 8192, 8192, 2986] dup_rows_per_dst=[2022, 3454, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[7960, 8192, 8154, 8192, 8192, 8192, 8192, 2986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=328552448 dispatch_meta_bytes=183352 combine_payload_bytes=328552448 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=328735800 ingress_bytes=548753408 link_bytes=548753408 token_bytes=657288248 weight_movement_bytes=220200960 total_nvlink_bytes=877489208 weight_to_token_ratio=0.335014 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=14336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.543724309e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60182 payload_rows_remote=52052 dup_rows=5354 dup_rows_remote=5342 dedup_ratio=0.081696 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1817, 0, 0, 2, 0, 0, 903, 4096, 0, 0, 0, 7, 4096, 0, 0, 4064, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1518, 0, 0, 3977, 0, 0] payload_rows_per_dst=[8056, 8192, 8130, 8192, 8192, 8192, 8192, 3036] dup_rows_per_dst=[3532, 1810, 12, 0, 0, 0, 0, 0] combine_rows_per_src=[8056, 8192, 8130, 8192, 8192, 8192, 8192, 3036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=371288064 dispatch_meta_bytes=217032 combine_payload_bytes=371288064 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=371505096 ingress_bytes=701589504 link_bytes=701589504 token_bytes=742793160 weight_movement_bytes=330301440 total_nvlink_bytes=1073094600 weight_to_token_ratio=0.444675 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=24576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.220179539e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=330975232 dispatch_meta_bytes=229376 combine_payload_bytes=330975232 prefetch_bytes=0 reduce_bytes=0 egress_bytes=331204608 ingress_bytes=330975232 link_bytes=331204608 token_bytes=662179840 weight_movement_bytes=0 total_nvlink_bytes=662179840 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=5.760192871e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=381638656 dispatch_meta_bytes=218560 combine_payload_bytes=381638656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=381857216 ingress_bytes=491739136 link_bytes=491739136 token_bytes=763495872 weight_movement_bytes=110100480 total_nvlink_bytes=873596352 weight_to_token_ratio=0.144206 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.552152347e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=371789824 dispatch_meta_bytes=229376 combine_payload_bytes=371789824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372019200 ingress_bytes=481890304 link_bytes=481890304 token_bytes=743809024 weight_movement_bytes=110100480 total_nvlink_bytes=853909504 weight_to_token_ratio=0.148023 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.380864960e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=373108736 dispatch_meta_bytes=229576 combine_payload_bytes=373108736 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=373338312 ingress_bytes=593309696 link_bytes=593309696 token_bytes=746447048 weight_movement_bytes=220200960 total_nvlink_bytes=966648008 weight_to_token_ratio=0.294999 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.031863144e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62674 payload_rows_remote=54482 dup_rows=2862 dup_rows_remote=2862 dedup_ratio=0.043671 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 4096, 2672, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 1429, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5344, 8192, 8192, 8178, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 2858, 0] combine_rows_per_src=[8192, 5344, 8192, 8192, 8178, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63668 payload_rows_remote=56688 dup_rows=1868 dup_rows_remote=1868 dedup_ratio=0.028503 router_tokens_per_expert=[0, 0, 87, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 0, 0, 2, 0, 0, 0, 0, 0, 3490, 0, 0, 0, 0, 203, 4096, 88, 0, 0, 0, 556, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7828, 7900, 6980, 8192, 8192] dup_rows_per_dst=[174, 0, 0, 0, 0, 0, 1694, 0] combine_rows_per_src=[8192, 8192, 8192, 7828, 7900, 6980, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62800 payload_rows_remote=54612 dup_rows=2736 dup_rows_remote=2720 dedup_ratio=0.041748 router_tokens_per_expert=[0, 0, 18, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 2, 0, 0, 0, 0, 2, 0, 3628, 0, 0, 0, 0, 163, 4096, 83, 0, 0, 0, 341, 4094, 0, 0, 0, 0, 4094, 0, 0] payload_rows_per_dst=[6660, 8190, 8192, 8094, 8028, 7256, 8192, 8188] dup_rows_per_dst=[28, 0, 0, 1360, 0, 4, 1328, 16] combine_rows_per_src=[6660, 8190, 8192, 8094, 8028, 7256, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=390526976 dispatch_meta_bytes=229376 combine_payload_bytes=390526976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=390756352 ingress_bytes=500627456 link_bytes=500627456 token_bytes=781283328 weight_movement_bytes=110100480 total_nvlink_bytes=891383808 weight_to_token_ratio=0.140923 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.706734851e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61176 payload_rows_remote=52984 dup_rows=4360 dup_rows_remote=4360 dedup_ratio=0.066528 router_tokens_per_expert=[0, 0, 106, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 7, 0, 0, 0, 0, 4, 0, 3500, 0, 0, 0, 0, 146, 4096, 42, 0, 0, 0, 439, 4096, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8030, 8072, 7338, 4970, 8190] dup_rows_per_dst=[212, 0, 2, 0, 0, 3316, 830, 0] combine_rows_per_src=[8192, 8192, 8192, 8030, 8072, 7338, 4970, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63792 payload_rows_remote=55600 dup_rows=1744 dup_rows_remote=240 dedup_ratio=0.026611 router_tokens_per_expert=[0, 0, 120, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4021, 0, 0, 1, 0, 0, 0, 0, 0, 3421, 0, 0, 0, 0, 163, 4096, 84, 0, 0, 0, 505, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7946, 8044, 6842, 8192, 8192] dup_rows_per_dst=[240, 0, 0, 0, 0, 0, 1504, 0] combine_rows_per_src=[8192, 8192, 8192, 7946, 8044, 6842, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=406339584 dispatch_meta_bytes=234224 combine_payload_bytes=406339584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406573808 ingress_bytes=516440064 link_bytes=516440064 token_bytes=812913392 weight_movement_bytes=110100480 total_nvlink_bytes=923013872 weight_to_token_ratio=0.135439 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.981742111e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58118 payload_rows_remote=49926 dup_rows=7418 dup_rows_remote=7184 dedup_ratio=0.113190 router_tokens_per_expert=[0, 0, 115, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 10, 0, 0, 0, 0, 5, 0, 3567, 0, 0, 0, 0, 161, 4096, 85, 0, 0, 0, 435, 4094, 0, 0, 0, 0, 4094, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8002, 7966, 8036, 1358, 8184] dup_rows_per_dst=[234, 4, 0, 0, 1582, 5594, 4, 0] combine_rows_per_src=[8192, 8192, 8188, 8002, 7966, 8036, 1358, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=391458816 dispatch_meta_bytes=229328 combine_payload_bytes=391458816 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=391688144 ingress_bytes=721760256 link_bytes=721760256 token_bytes=783146960 weight_movement_bytes=330301440 total_nvlink_bytes=1113448400 weight_to_token_ratio=0.421762 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.255259794e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64118 payload_rows_remote=56148 dup_rows=1418 dup_rows_remote=1418 dedup_ratio=0.021637 router_tokens_per_expert=[0, 0, 122, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 3536, 0, 0, 0, 0, 226, 4096, 18, 0, 0, 0, 343, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8116, 7970, 7072, 8192, 8192] dup_rows_per_dst=[244, 0, 0, 0, 0, 0, 1174, 0] combine_rows_per_src=[8192, 8192, 8192, 8116, 7970, 7072, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63564 payload_rows_remote=55372 dup_rows=1972 dup_rows_remote=1970 dedup_ratio=0.030090 router_tokens_per_expert=[0, 0, 84, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 0, 4, 0, 0, 0, 0, 2, 0, 3292, 0, 0, 0, 0, 157, 4096, 72, 0, 0, 0, 670, 4094, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8086, 7938, 6584, 8192, 8190] dup_rows_per_dst=[168, 0, 2, 0, 0, 4, 1798, 0] combine_rows_per_src=[8192, 8190, 8192, 8086, 7938, 6584, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=379789312 dispatch_meta_bytes=229376 combine_payload_bytes=379789312 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=380018688 ingress_bytes=599990272 link_bytes=599990272 token_bytes=759808000 weight_movement_bytes=220200960 total_nvlink_bytes=980008960 weight_to_token_ratio=0.289811 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.043481764e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=398540800 dispatch_meta_bytes=223360 combine_payload_bytes=398540800 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=398764160 ingress_bytes=728842240 link_bytes=728842240 token_bytes=797304960 weight_movement_bytes=330301440 total_nvlink_bytes=1127606400 weight_to_token_ratio=0.414272 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=26880 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.267576529e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=357869568 dispatch_meta_bytes=228440 combine_payload_bytes=357869568 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=358098008 ingress_bytes=798271488 link_bytes=798271488 token_bytes=715967576 weight_movement_bytes=440401920 total_nvlink_bytes=1156369496 weight_to_token_ratio=0.615114 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.388325411e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=402468864 dispatch_meta_bytes=230264 combine_payload_bytes=402468864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402699128 ingress_bytes=512569344 link_bytes=512569344 token_bytes=805167992 weight_movement_bytes=110100480 total_nvlink_bytes=915268472 weight_to_token_ratio=0.136742 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.914423924e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=396906496 dispatch_meta_bytes=229368 combine_payload_bytes=396906496 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=397135864 ingress_bytes=507006976 link_bytes=507006976 token_bytes=794042360 weight_movement_bytes=110100480 total_nvlink_bytes=904142840 weight_to_token_ratio=0.138658 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.817685196e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60940 payload_rows_remote=52748 dup_rows=4596 dup_rows_remote=4584 dedup_ratio=0.070129 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1502, 0, 0, 0, 0, 0, 881, 4096, 0, 0, 0, 0, 4096, 0, 0, 4028, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1946, 0, 0, 3925, 0, 0] payload_rows_per_dst=[8032, 8192, 8056, 8192, 8192, 8192, 8192, 3892] dup_rows_per_dst=[2822, 1762, 0, 12, 0, 0, 0, 0] combine_rows_per_src=[8032, 8192, 8056, 8192, 8192, 8192, 8192, 3892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61300 payload_rows_remote=53108 dup_rows=4236 dup_rows_remote=4218 dedup_ratio=0.064636 router_tokens_per_expert=[313, 0, 0, 0, 0, 4096, 0, 0, 0, 1481, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3988, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 2283, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5422, 8192, 8192, 7976, 8192, 6942, 8192] dup_rows_per_dst=[626, 772, 0, 0, 0, 18, 2820, 0] combine_rows_per_src=[8192, 5422, 8192, 8192, 7976, 8192, 6942, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57240 payload_rows_remote=49048 dup_rows=8296 dup_rows_remote=8296 dedup_ratio=0.126587 router_tokens_per_expert=[96, 0, 0, 0, 0, 4094, 0, 0, 0, 1267, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3960, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 1, 0, 0, 2757, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5310, 3022, 8192, 8188, 7952, 8192, 8192, 8192] dup_rows_per_dst=[174, 284, 0, 0, 2096, 6, 5736, 0] combine_rows_per_src=[5310, 3022, 8192, 8188, 7952, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=378097664 dispatch_meta_bytes=229328 combine_payload_bytes=378097664 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=378326992 ingress_bytes=598298624 link_bytes=598298624 token_bytes=756424656 weight_movement_bytes=220200960 total_nvlink_bytes=976625616 weight_to_token_ratio=0.291108 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.040539710e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=380678144 dispatch_meta_bytes=229304 combine_payload_bytes=380678144 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=380907448 ingress_bytes=600879104 link_bytes=600879104 token_bytes=761585592 weight_movement_bytes=220200960 total_nvlink_bytes=981786552 weight_to_token_ratio=0.289135 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.045027589e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63276 payload_rows_remote=55084 dup_rows=2260 dup_rows_remote=566 dedup_ratio=0.034485 router_tokens_per_expert=[169, 0, 0, 0, 0, 4094, 0, 0, 0, 847, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3826, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3218, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7652, 8192, 6472, 8192] dup_rows_per_dst=[334, 1694, 0, 0, 0, 0, 232, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7652, 8192, 6472, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60112 payload_rows_remote=51920 dup_rows=5424 dup_rows_remote=264 dedup_ratio=0.082764 router_tokens_per_expert=[120, 0, 0, 0, 0, 4096, 0, 0, 0, 1470, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4010, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 12, 0, 0, 2466, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 2940, 8192, 8192, 8020, 8192, 8192, 8192] dup_rows_per_dst=[240, 0, 0, 0, 0, 24, 5160, 0] combine_rows_per_src=[8192, 2940, 8192, 8192, 8020, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62776 payload_rows_remote=54584 dup_rows=2760 dup_rows_remote=1900 dedup_ratio=0.042114 router_tokens_per_expert=[385, 0, 0, 0, 0, 4087, 0, 0, 0, 875, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3954, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 2863, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 7906, 8078, 5834, 8192] dup_rows_per_dst=[860, 1752, 2, 6, 0, 2, 138, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 7906, 8078, 5834, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62092 payload_rows_remote=54002 dup_rows=3444 dup_rows_remote=3444 dedup_ratio=0.052551 router_tokens_per_expert=[188, 0, 0, 0, 0, 4096, 0, 0, 0, 1519, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4045, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 2296, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8090, 8192, 4850, 8192] dup_rows_per_dst=[376, 3038, 0, 0, 0, 8, 22, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8090, 8192, 4850, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=351576064 dispatch_meta_bytes=229376 combine_payload_bytes=351576064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351805440 ingress_bytes=461676544 link_bytes=461676544 token_bytes=703381504 weight_movement_bytes=110100480 total_nvlink_bytes=813481984 weight_to_token_ratio=0.156530 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.029314428e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63080 payload_rows_remote=54890 dup_rows=2456 dup_rows_remote=2456 dedup_ratio=0.037476 router_tokens_per_expert=[53, 0, 0, 0, 0, 4094, 0, 0, 0, 1109, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3878, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 3024, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 7756, 8192, 6174, 8192] dup_rows_per_dst=[102, 2218, 0, 0, 0, 10, 126, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 7756, 8192, 6174, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7330055888 prefetch_bytes=880803840 reduce_bytes=660602880 weight_movement_bytes=1541406720 egress_bytes=3666175696 ingress_bytes=5205286912 link_bytes=5205286912 total_nvlink_bytes=8871462608 weight_to_token_ratio=0.210286 prefetch_share_of_ingress=0.169213 dedup_ratio=0.101776 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=6 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6350 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.052850063e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=394842112 dispatch_meta_bytes=222600 combine_payload_bytes=394842112 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=395064712 ingress_bytes=615043072 link_bytes=615043072 token_bytes=789906824 weight_movement_bytes=220200960 total_nvlink_bytes=1010107784 weight_to_token_ratio=0.278768 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=22144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.069661059e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=372162560 dispatch_meta_bytes=208736 combine_payload_bytes=372162560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372371296 ingress_bytes=482263040 link_bytes=482263040 token_bytes=744533856 weight_movement_bytes=110100480 total_nvlink_bytes=854634336 weight_to_token_ratio=0.147878 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=21120 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.387347452e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=391258112 dispatch_meta_bytes=225936 combine_payload_bytes=391258112 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=391484048 ingress_bytes=721559552 link_bytes=721559552 token_bytes=782742160 weight_movement_bytes=330301440 total_nvlink_bytes=1113043600 weight_to_token_ratio=0.421980 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.254910737e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6878059256 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=3440114424 ingress_bytes=5089452032 link_bytes=5089452032 total_nvlink_bytes=8529566456 weight_to_token_ratio=0.240112 prefetch_share_of_ingress=0.216331 dedup_ratio=0.146100 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6223 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.851394155e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=387086336 dispatch_meta_bytes=229784 combine_payload_bytes=387086336 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=387316120 ingress_bytes=717387776 link_bytes=717387776 token_bytes=774402456 weight_movement_bytes=330301440 total_nvlink_bytes=1104703896 weight_to_token_ratio=0.426524 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.247655332e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=393451520 dispatch_meta_bytes=229384 combine_payload_bytes=393451520 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=393680904 ingress_bytes=613652480 link_bytes=613652480 token_bytes=787132424 weight_movement_bytes=220200960 total_nvlink_bytes=1007333384 weight_to_token_ratio=0.279751 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.067242591e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7219579768 prefetch_bytes=1321205760 reduce_bytes=880803840 weight_movement_bytes=2202009600 egress_bytes=3610893176 ingress_bytes=5810696192 link_bytes=5810696192 total_nvlink_bytes=9421589368 weight_to_token_ratio=0.305005 prefetch_share_of_ingress=0.227375 dedup_ratio=0.112576 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=8 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.010575637e-04 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63580 payload_rows_remote=55506 dup_rows=1956 dup_rows_remote=1956 dedup_ratio=0.029846 router_tokens_per_expert=[0, 0, 116, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 4, 0, 0, 0, 0, 0, 0, 3255, 0, 0, 0, 0, 209, 4096, 176, 0, 0, 0, 477, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8074, 8036, 6510, 8192, 8192] dup_rows_per_dst=[232, 0, 0, 0, 0, 0, 1724, 0] combine_rows_per_src=[8192, 8192, 8192, 8074, 8036, 6510, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7166496176 prefetch_bytes=990904320 reduce_bytes=1321205760 weight_movement_bytes=2312110080 egress_bytes=3584345520 ingress_bytes=5894260736 link_bytes=5894490112 total_nvlink_bytes=9478606256 weight_to_token_ratio=0.322628 prefetch_share_of_ingress=0.168113 dedup_ratio=0.115787 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=12 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051103 gemm_inner_pad_upper_bound=5842 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.025148778e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6960951160 prefetch_bytes=990904320 reduce_bytes=1871708160 weight_movement_bytes=2862612480 egress_bytes=3481560952 ingress_bytes=6342002688 link_bytes=6342002688 total_nvlink_bytes=9823563640 weight_to_token_ratio=0.411239 prefetch_share_of_ingress=0.156245 dedup_ratio=0.136960 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=17 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6223 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.102978575e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7457636576 prefetch_bytes=880803840 reduce_bytes=1101004800 weight_movement_bytes=1981808640 egress_bytes=3729961184 ingress_bytes=5709484032 link_bytes=5709484032 total_nvlink_bytes=9439445216 weight_to_token_ratio=0.265742 prefetch_share_of_ingress=0.154270 dedup_ratio=0.087653 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=10 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050735 gemm_inner_pad_upper_bound=5842 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.929731780e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7383262928 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=3692775120 ingress_bytes=5231894528 link_bytes=5232123904 total_nvlink_bytes=8924669648 weight_to_token_ratio=0.208770 prefetch_share_of_ingress=0.189397 dedup_ratio=0.093231 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051838 gemm_inner_pad_upper_bound=5588 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.099524006e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=397867008 dispatch_meta_bytes=229848 combine_payload_bytes=397867008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398096856 ingress_bytes=507967488 link_bytes=507967488 token_bytes=795963864 weight_movement_bytes=110100480 total_nvlink_bytes=906064344 weight_to_token_ratio=0.138323 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.834390079e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61870 payload_rows_remote=53678 dup_rows=3666 dup_rows_remote=3666 dedup_ratio=0.055939 router_tokens_per_expert=[85, 0, 0, 0, 0, 4096, 0, 0, 0, 1717, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4044, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 2184, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8088, 8192, 4630, 8192] dup_rows_per_dst=[170, 3434, 0, 0, 0, 46, 16, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8088, 8192, 4630, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=384763904 dispatch_meta_bytes=229376 combine_payload_bytes=384763904 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=384993280 ingress_bytes=604964864 link_bytes=604964864 token_bytes=769757184 weight_movement_bytes=220200960 total_nvlink_bytes=989958144 weight_to_token_ratio=0.286065 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.052133398e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7470854392 prefetch_bytes=770703360 reduce_bytes=990904320 weight_movement_bytes=1761607680 egress_bytes=3736570104 ingress_bytes=5495891968 link_bytes=5495891968 total_nvlink_bytes=9232462072 weight_to_token_ratio=0.235797 prefetch_share_of_ingress=0.140233 dedup_ratio=0.089896 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=9 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051471 gemm_inner_pad_upper_bound=6096 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.558260051e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none + [2026-08-26 18:09:53] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 364.7 | throughput per GPU (TFLOP/s/GPU): 338.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.896148E+00 | loss scale: 1.0 | grad norm: 4.603 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44750 payload_rows_remote=39706 dup_rows=20786 dup_rows_remote=18722 dedup_ratio=0.317169 router_tokens_per_expert=[1074, 1327, 1324, 903, 106, 1083, 383, 52, 419, 1184, 1225, 333, 6, 1262, 32, 1035, 1025, 553, 32, 64, 1141, 390, 96, 536, 221, 668, 498, 845, 486, 137, 380, 1, 579, 421, 885, 40, 502, 1, 1226, 147, 585, 475, 373, 1192, 6, 413, 89, 421, 233, 190, 181, 695, 247, 731, 36, 1191, 498, 554, 690, 262, 167, 594, 304, 19] payload_rows_per_dst=[5170, 5994, 5606, 5610, 5260, 5044, 6422, 5644] dup_rows_per_dst=[2532, 2410, 2068, 3010, 2562, 2064, 3240, 2900] combine_rows_per_src=[5170, 5994, 5606, 5610, 5260, 5044, 6422, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43736 payload_rows_remote=36742 dup_rows=21800 dup_rows_remote=16004 dedup_ratio=0.332642 router_tokens_per_expert=[1444, 1244, 930, 612, 59, 1002, 245, 94, 761, 1476, 931, 328, 7, 1020, 30, 1842, 1119, 844, 30, 42, 819, 296, 203, 811, 340, 615, 381, 561, 257, 118, 673, 1, 541, 488, 1434, 7, 453, 9, 817, 160, 338, 778, 605, 1113, 10, 793, 39, 497, 320, 221, 244, 301, 135, 863, 93, 694, 369, 656, 562, 295, 252, 419, 127, 0] payload_rows_per_dst=[6622, 6994, 5904, 4544, 5308, 5670, 4488, 4206] dup_rows_per_dst=[4638, 5796, 2424, 1348, 2510, 2676, 1254, 1154] combine_rows_per_src=[6622, 6994, 5904, 4544, 5308, 5670, 4488, 4206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43724 payload_rows_remote=37346 dup_rows=21812 dup_rows_remote=17940 dedup_ratio=0.332825 router_tokens_per_expert=[1360, 1332, 1244, 525, 38, 1122, 339, 97, 499, 1714, 916, 353, 2, 692, 53, 1401, 888, 851, 12, 42, 1205, 271, 672, 646, 298, 596, 572, 691, 386, 244, 328, 10, 703, 548, 707, 9, 693, 0, 839, 104, 227, 999, 501, 642, 1, 575, 59, 296, 159, 218, 256, 334, 144, 1131, 143, 670, 414, 808, 1127, 358, 263, 172, 268, 1] payload_rows_per_dst=[3378, 3226, 7084, 5692, 5474, 6144, 6348, 6378] dup_rows_per_dst=[864, 420, 4578, 3278, 3116, 3258, 2426, 3872] combine_rows_per_src=[3378, 3226, 7084, 5692, 5474, 6144, 6348, 6378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43868 payload_rows_remote=36864 dup_rows=21668 dup_rows_remote=17816 dedup_ratio=0.330627 router_tokens_per_expert=[1581, 1426, 929, 586, 303, 788, 431, 61, 244, 1305, 1113, 395, 4, 901, 11, 1527, 851, 370, 12, 43, 761, 485, 516, 477, 170, 1107, 353, 625, 303, 246, 576, 34, 854, 347, 1008, 2, 601, 2, 1053, 82, 187, 644, 364, 607, 0, 959, 99, 373, 219, 195, 339, 677, 280, 993, 74, 1225, 346, 989, 439, 162, 478, 373, 262, 1] payload_rows_per_dst=[4062, 3590, 5620, 6240, 5798, 5822, 7004, 5732] dup_rows_per_dst=[1532, 690, 2012, 3750, 3902, 2952, 3852, 2978] combine_rows_per_src=[4062, 3590, 5620, 6240, 5798, 5822, 7004, 5732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45044 payload_rows_remote=40052 dup_rows=20492 dup_rows_remote=18038 dedup_ratio=0.312683 router_tokens_per_expert=[1346, 1329, 827, 837, 132, 1100, 457, 93, 296, 1181, 1228, 334, 4, 1242, 29, 1316, 819, 714, 32, 25, 888, 291, 589, 694, 343, 773, 367, 919, 329, 180, 393, 1, 546, 549, 765, 5, 549, 0, 1132, 177, 491, 513, 315, 837, 0, 727, 82, 527, 160, 102, 309, 263, 179, 1009, 117, 784, 443, 771, 577, 448, 599, 443, 240, 1] payload_rows_per_dst=[4980, 6198, 5774, 5916, 4992, 4978, 6180, 6026] dup_rows_per_dst=[1912, 2700, 2330, 3386, 2454, 2006, 2324, 3380] combine_rows_per_src=[4980, 6198, 5774, 5916, 4992, 4978, 6180, 6026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44134 payload_rows_remote=38898 dup_rows=21402 dup_rows_remote=19792 dedup_ratio=0.326569 router_tokens_per_expert=[1542, 881, 1088, 752, 219, 992, 508, 183, 255, 1358, 729, 347, 4, 1178, 32, 955, 678, 972, 8, 39, 642, 289, 363, 432, 428, 987, 276, 577, 145, 247, 641, 4, 720, 549, 674, 17, 717, 3, 980, 199, 278, 840, 277, 1338, 2, 936, 40, 501, 95, 114, 562, 418, 407, 1011, 90, 571, 395, 659, 919, 174, 306, 996, 229, 0] payload_rows_per_dst=[6504, 6078, 5236, 5202, 5020, 5732, 5104, 5258] dup_rows_per_dst=[4996, 3456, 1610, 1778, 2698, 2692, 1892, 2280] combine_rows_per_src=[6504, 6078, 5236, 5202, 5020, 5732, 5104, 5258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45094 payload_rows_remote=39176 dup_rows=20442 dup_rows_remote=16996 dedup_ratio=0.311920 router_tokens_per_expert=[1597, 1026, 1254, 929, 333, 351, 477, 30, 280, 1473, 895, 178, 3, 1293, 10, 1021, 1078, 742, 16, 29, 625, 391, 261, 835, 136, 1073, 335, 616, 279, 122, 517, 7, 659, 315, 1144, 8, 755, 0, 964, 211, 301, 727, 427, 1101, 1, 781, 71, 628, 118, 208, 166, 368, 333, 990, 63, 600, 764, 1127, 413, 201, 328, 551, 233, 0] payload_rows_per_dst=[4824, 5532, 5728, 5918, 5372, 5600, 5704, 6416] dup_rows_per_dst=[1924, 1828, 2226, 3446, 2740, 2474, 2040, 3764] combine_rows_per_src=[4824, 5532, 5728, 5918, 5372, 5600, 5704, 6416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284612608 dispatch_meta_bytes=233712 combine_payload_bytes=284612608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=284846320 ingress_bytes=394713088 link_bytes=394713088 token_bytes=569458928 weight_movement_bytes=110100480 total_nvlink_bytes=679559408 weight_to_token_ratio=0.193342 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.864709792e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44324 payload_rows_remote=37612 dup_rows=21212 dup_rows_remote=16326 dedup_ratio=0.323669 router_tokens_per_expert=[1417, 1148, 1088, 479, 133, 632, 267, 105, 331, 975, 1011, 380, 16, 1663, 16, 1450, 1164, 300, 16, 65, 899, 310, 468, 446, 393, 829, 276, 741, 211, 254, 495, 10, 774, 401, 1025, 22, 879, 0, 735, 329, 617, 983, 347, 1244, 10, 727, 11, 273, 66, 194, 319, 350, 357, 1361, 71, 911, 316, 598, 663, 392, 235, 444, 125, 1] payload_rows_per_dst=[6712, 6634, 5456, 4826, 5298, 5718, 5476, 4204] dup_rows_per_dst=[4886, 3990, 1880, 1592, 3032, 2706, 1782, 1344] combine_rows_per_src=[6712, 6634, 5456, 4826, 5298, 5718, 5476, 4204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=263366656 dispatch_meta_bytes=210984 combine_payload_bytes=263366656 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=263577640 ingress_bytes=703768576 link_bytes=703768576 token_bytes=526944296 weight_movement_bytes=440401920 total_nvlink_bytes=967346216 weight_to_token_ratio=0.835766 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.223969304e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=267696128 dispatch_meta_bytes=221144 combine_payload_bytes=267696128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=267917272 ingress_bytes=377796608 link_bytes=377796608 token_bytes=535613400 weight_movement_bytes=110100480 total_nvlink_bytes=645713880 weight_to_token_ratio=0.205560 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.570504382e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=264241152 dispatch_meta_bytes=218720 combine_payload_bytes=264241152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=264459872 ingress_bytes=374341632 link_bytes=374341632 token_bytes=528701024 weight_movement_bytes=110100480 total_nvlink_bytes=638801504 weight_to_token_ratio=0.208247 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.510416667e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=287092736 dispatch_meta_bytes=232360 combine_payload_bytes=287092736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=287325096 ingress_bytes=397193216 link_bytes=397193216 token_bytes=574417832 weight_movement_bytes=110100480 total_nvlink_bytes=684518312 weight_to_token_ratio=0.191673 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.907843297e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278820864 dispatch_meta_bytes=234760 combine_payload_bytes=278820864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279055624 ingress_bytes=388921344 link_bytes=388921344 token_bytes=557876488 weight_movement_bytes=110100480 total_nvlink_bytes=667976968 weight_to_token_ratio=0.197356 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763981838e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280813568 dispatch_meta_bytes=224688 combine_payload_bytes=280813568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281038256 ingress_bytes=390914048 link_bytes=390914048 token_bytes=561851824 weight_movement_bytes=110100480 total_nvlink_bytes=671952304 weight_to_token_ratio=0.195960 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.798638238e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=269602816 dispatch_meta_bytes=215752 combine_payload_bytes=269602816 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=269818568 ingress_bytes=710004736 link_bytes=710004736 token_bytes=539421384 weight_movement_bytes=440401920 total_nvlink_bytes=979823304 weight_to_token_ratio=0.816434 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.234815012e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55598 payload_rows_remote=48008 dup_rows=9938 dup_rows_remote=9938 dedup_ratio=0.151642 router_tokens_per_expert=[0, 0, 0, 1298, 3899, 0, 44, 3933, 1180, 19, 49, 0, 0, 2, 0, 156, 0, 13, 0, 0, 3527, 417, 0, 1, 0, 0, 2784, 0, 21, 2942, 0, 2, 0, 0, 0, 0, 3, 1, 0, 23, 0, 2, 3793, 0, 0, 0, 0, 0, 0, 249, 3296, 0, 160, 0, 0, 0, 1788, 0, 16, 0, 3, 2, 9, 3136] payload_rows_per_dst=[5468, 6092, 7238, 7066, 7808, 7590, 6844, 7492] dup_rows_per_dst=[1338, 1656, 678, 3240, 44, 0, 566, 2416] combine_rows_per_src=[5468, 6092, 7238, 7066, 7808, 7590, 6844, 7492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56180 payload_rows_remote=49212 dup_rows=9356 dup_rows_remote=6684 dedup_ratio=0.142761 router_tokens_per_expert=[0, 0, 0, 1453, 3973, 0, 85, 3918, 1472, 26, 152, 0, 0, 0, 0, 109, 0, 0, 0, 10, 3717, 433, 0, 0, 0, 0, 2087, 0, 0, 3061, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3960, 0, 0, 0, 2, 0, 0, 282, 3109, 0, 231, 0, 0, 0, 1241, 0, 0, 0, 5, 0, 10, 3427] payload_rows_per_dst=[8056, 6968, 7548, 4174, 7956, 7920, 6940, 6618] dup_rows_per_dst=[2856, 2672, 772, 0, 0, 4, 1450, 1602] combine_rows_per_src=[8056, 6968, 7548, 4174, 7956, 7920, 6940, 6618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51160 payload_rows_remote=44990 dup_rows=14376 dup_rows_remote=12964 dedup_ratio=0.219360 router_tokens_per_expert=[0, 0, 0, 1805, 3691, 0, 59, 3774, 1502, 32, 463, 0, 0, 16, 0, 214, 0, 1, 0, 36, 3461, 800, 0, 0, 0, 0, 2198, 0, 1, 3261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4059, 0, 0, 0, 9, 0, 0, 345, 3019, 0, 226, 0, 0, 0, 1183, 0, 0, 1, 15, 0, 3, 2589] payload_rows_per_dst=[3676, 3830, 7374, 8030, 7496, 8118, 6466, 6170] dup_rows_per_dst=[52, 624, 1222, 7342, 2992, 18, 714, 1412] combine_rows_per_src=[3676, 3830, 7374, 8030, 7496, 8118, 6466, 6170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51774 payload_rows_remote=44548 dup_rows=13762 dup_rows_remote=13040 dedup_ratio=0.209991 router_tokens_per_expert=[0, 0, 0, 861, 4031, 0, 53, 3949, 1654, 19, 74, 0, 0, 0, 0, 48, 0, 0, 0, 5, 3462, 582, 0, 0, 0, 0, 1976, 1, 3, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 0, 0, 0, 0, 279, 3466, 0, 229, 0, 0, 0, 1838, 0, 2, 0, 4, 0, 0, 3661] payload_rows_per_dst=[1828, 3484, 7232, 8174, 8062, 7828, 7226, 7940] dup_rows_per_dst=[0, 106, 866, 8998, 0, 0, 722, 3070] combine_rows_per_src=[1828, 3484, 7232, 8174, 8062, 7828, 7226, 7940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57754 payload_rows_remote=49728 dup_rows=7782 dup_rows_remote=7782 dedup_ratio=0.118744 router_tokens_per_expert=[0, 0, 0, 1207, 4012, 0, 60, 3962, 1372, 10, 47, 0, 0, 0, 0, 1, 0, 1, 0, 16, 3657, 295, 1, 0, 0, 0, 2524, 0, 3, 2920, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 3943, 0, 0, 0, 6, 0, 0, 272, 3410, 0, 207, 0, 0, 0, 1613, 0, 0, 0, 0, 1, 0, 3226] payload_rows_per_dst=[8060, 6704, 7424, 5048, 8026, 7898, 7138, 7456] dup_rows_per_dst=[2398, 1996, 516, 6, 0, 2, 640, 2224] combine_rows_per_src=[8060, 6704, 7424, 5048, 8026, 7898, 7138, 7456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56022 payload_rows_remote=49094 dup_rows=9514 dup_rows_remote=8628 dedup_ratio=0.145172 router_tokens_per_expert=[0, 0, 0, 1042, 3879, 0, 53, 3821, 2457, 18, 259, 0, 0, 0, 0, 82, 2, 3, 0, 30, 3316, 556, 0, 0, 0, 0, 2087, 0, 2, 3061, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4002, 0, 0, 1, 0, 0, 0, 464, 3636, 0, 312, 0, 1, 0, 570, 0, 0, 3, 18, 1, 17, 3073] payload_rows_per_dst=[7910, 7170, 6928, 4174, 7760, 8006, 7494, 6580] dup_rows_per_dst=[1922, 4584, 886, 4, 2, 0, 1332, 784] combine_rows_per_src=[7910, 7170, 6928, 4174, 7760, 8006, 7494, 6580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=344121344 dispatch_meta_bytes=231784 combine_payload_bytes=344121344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=344353128 ingress_bytes=454221824 link_bytes=454221824 token_bytes=688474472 weight_movement_bytes=110100480 total_nvlink_bytes=798574952 weight_to_token_ratio=0.159919 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.899664586e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=352751616 dispatch_meta_bytes=223584 combine_payload_bytes=352751616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352975200 ingress_bytes=462852096 link_bytes=462852096 token_bytes=705726816 weight_movement_bytes=110100480 total_nvlink_bytes=815827296 weight_to_token_ratio=0.156010 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.049759210e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=322488320 dispatch_meta_bytes=231816 combine_payload_bytes=322488320 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=322720136 ingress_bytes=762890240 link_bytes=762890240 token_bytes=645208456 weight_movement_bytes=440401920 total_nvlink_bytes=1085610376 weight_to_token_ratio=0.682573 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=21632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.326791601e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56578 payload_rows_remote=52142 dup_rows=8958 dup_rows_remote=8958 dedup_ratio=0.136688 router_tokens_per_expert=[0, 0, 0, 1005, 3897, 0, 63, 3832, 1932, 25, 33, 0, 2, 0, 0, 10, 0, 0, 0, 5, 3512, 498, 0, 0, 0, 0, 2218, 0, 0, 2987, 0, 0, 0, 2, 0, 0, 0, 7, 0, 3, 0, 0, 3864, 0, 0, 0, 5, 0, 0, 254, 3503, 2, 252, 0, 0, 0, 1768, 0, 0, 0, 7, 0, 2, 3080] payload_rows_per_dst=[7874, 6968, 7188, 4436, 7804, 7732, 7260, 7316] dup_rows_per_dst=[1926, 3010, 842, 0, 14, 6, 762, 2398] combine_rows_per_src=[7874, 6968, 7188, 4436, 7804, 7732, 7260, 7316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=319320064 dispatch_meta_bytes=230352 combine_payload_bytes=319320064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=319550416 ingress_bytes=429420544 link_bytes=429420544 token_bytes=638870480 weight_movement_bytes=110100480 total_nvlink_bytes=748970960 weight_to_token_ratio=0.172336 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=27264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.468329536e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=356450304 dispatch_meta_bytes=230040 combine_payload_bytes=356450304 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=356680344 ingress_bytes=576651264 link_bytes=576651264 token_bytes=713130648 weight_movement_bytes=220200960 total_nvlink_bytes=933331608 weight_to_token_ratio=0.308781 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.002891391e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52026 payload_rows_remote=43952 dup_rows=13510 dup_rows_remote=9412 dedup_ratio=0.206146 router_tokens_per_expert=[0, 0, 0, 1455, 3786, 0, 56, 3926, 1902, 16, 65, 0, 0, 0, 0, 28, 0, 1, 0, 0, 3550, 178, 0, 0, 0, 0, 2421, 0, 1, 2825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3805, 0, 2, 0, 1, 0, 1, 275, 3712, 0, 199, 0, 0, 0, 1016, 0, 5, 1, 1, 0, 6, 3533] payload_rows_per_dst=[8074, 6788, 7292, 4844, 7572, 7730, 7674, 2052] dup_rows_per_dst=[4098, 2884, 1530, 0, 0, 2684, 2308, 6] combine_rows_per_src=[8074, 6788, 7292, 4844, 7572, 7730, 7674, 2052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=351905792 dispatch_meta_bytes=230888 combine_payload_bytes=351905792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352136680 ingress_bytes=462006272 link_bytes=462006272 token_bytes=704042472 weight_movement_bytes=110100480 total_nvlink_bytes=814142952 weight_to_token_ratio=0.156383 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.035048940e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=373753856 dispatch_meta_bytes=244400 combine_payload_bytes=373753856 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=373998256 ingress_bytes=593954816 link_bytes=593954816 token_bytes=747752112 weight_movement_bytes=220200960 total_nvlink_bytes=967953072 weight_to_token_ratio=0.294484 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.032985114e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=315047936 dispatch_meta_bytes=213456 combine_payload_bytes=315047936 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=315261392 ingress_bytes=755449856 link_bytes=755449856 token_bytes=630309328 weight_movement_bytes=440401920 total_nvlink_bytes=1070711248 weight_to_token_ratio=0.698708 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=21632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.313851550e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63236 payload_rows_remote=55142 dup_rows=2300 dup_rows_remote=2292 dedup_ratio=0.035095 router_tokens_per_expert=[0, 0, 0, 4014, 14, 0, 4091, 0, 0, 3861, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 4083, 88, 0, 0, 0, 764, 4096, 0, 220, 0, 0, 0, 0, 0, 0, 0, 82, 3263, 9, 4022, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4072, 0, 0, 0, 0, 12] payload_rows_per_dst=[8028, 7722, 8176, 8192, 6694, 8094, 8182, 8148] dup_rows_per_dst=[28, 0, 256, 1968, 14, 8, 0, 26] combine_rows_per_src=[8028, 7722, 8176, 8192, 6694, 8094, 8182, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63404 payload_rows_remote=55626 dup_rows=2132 dup_rows_remote=2132 dedup_ratio=0.032532 router_tokens_per_expert=[0, 0, 0, 4054, 0, 0, 4096, 0, 0, 3889, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 77, 0, 0, 0, 771, 4096, 0, 184, 0, 0, 0, 0, 0, 0, 0, 177, 3167, 0, 4048, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 1] payload_rows_per_dst=[8108, 7778, 8192, 8192, 6650, 8100, 8192, 8192] dup_rows_per_dst=[0, 0, 170, 1910, 38, 4, 0, 10] combine_rows_per_src=[8108, 7778, 8192, 8192, 6650, 8100, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58684 payload_rows_remote=50498 dup_rows=6852 dup_rows_remote=6834 dedup_ratio=0.104553 router_tokens_per_expert=[0, 0, 0, 3978, 0, 0, 4096, 0, 0, 3768, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 4094, 188, 0, 0, 0, 822, 4096, 0, 636, 0, 0, 0, 4, 0, 0, 0, 175, 2679, 0, 4083, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 9] payload_rows_per_dst=[7940, 7536, 7050, 3422, 8192, 8166, 8192, 8186] dup_rows_per_dst=[0, 0, 406, 220, 5716, 476, 16, 18] combine_rows_per_src=[7940, 7536, 7050, 3422, 8192, 8166, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63706 payload_rows_remote=56632 dup_rows=1830 dup_rows_remote=1800 dedup_ratio=0.027924 router_tokens_per_expert=[0, 0, 0, 3976, 1, 0, 4096, 0, 0, 3897, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 4092, 86, 0, 0, 0, 586, 4096, 0, 173, 1, 0, 0, 1, 0, 0, 0, 63, 3487, 1, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 9] payload_rows_per_dst=[7954, 7794, 8188, 8192, 7074, 8150, 8192, 8162] dup_rows_per_dst=[0, 0, 262, 1520, 30, 0, 0, 18] combine_rows_per_src=[7954, 7794, 8188, 8192, 7074, 8150, 8192, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60828 payload_rows_remote=52638 dup_rows=4708 dup_rows_remote=4708 dedup_ratio=0.071838 router_tokens_per_expert=[0, 0, 0, 4037, 2, 0, 4095, 0, 0, 3893, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 4096, 80, 0, 0, 0, 498, 4096, 0, 202, 0, 0, 0, 0, 0, 0, 0, 18, 3494, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 32] payload_rows_per_dst=[8074, 7786, 8192, 4800, 7414, 8184, 8190, 8188] dup_rows_per_dst=[4, 0, 236, 886, 3516, 0, 0, 66] combine_rows_per_src=[8074, 7786, 8192, 4800, 7414, 8184, 8190, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=395257856 dispatch_meta_bytes=229736 combine_payload_bytes=395257856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=395487592 ingress_bytes=505358336 link_bytes=505358336 token_bytes=790745448 weight_movement_bytes=110100480 total_nvlink_bytes=900845928 weight_to_token_ratio=0.139236 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.789012635e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62448 payload_rows_remote=54262 dup_rows=3088 dup_rows_remote=2698 dedup_ratio=0.047119 router_tokens_per_expert=[0, 0, 0, 3984, 3, 0, 4094, 0, 0, 3902, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 4093, 141, 0, 0, 0, 717, 4096, 0, 567, 1, 0, 0, 2, 0, 0, 0, 174, 2795, 0, 4028, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 22] payload_rows_per_dst=[7970, 7804, 8186, 8192, 5866, 8056, 8190, 8184] dup_rows_per_dst=[4, 0, 390, 2570, 76, 4, 0, 44] combine_rows_per_src=[7970, 7804, 8186, 8192, 5866, 8056, 8190, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=398727168 dispatch_meta_bytes=231032 combine_payload_bytes=398727168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398958200 ingress_bytes=508827648 link_bytes=508827648 token_bytes=797685368 weight_movement_bytes=110100480 total_nvlink_bytes=907785848 weight_to_token_ratio=0.138025 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.849349676e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=361969664 dispatch_meta_bytes=229328 combine_payload_bytes=361969664 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=362198992 ingress_bytes=472070144 link_bytes=472070144 token_bytes=724168656 weight_movement_bytes=110100480 total_nvlink_bytes=834269136 weight_to_token_ratio=0.152037 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.210076226e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=405938176 dispatch_meta_bytes=233728 combine_payload_bytes=405938176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406171904 ingress_bytes=516038656 link_bytes=516038656 token_bytes=812110080 weight_movement_bytes=110100480 total_nvlink_bytes=922210560 weight_to_token_ratio=0.135573 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=25984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.974760966e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=377309184 dispatch_meta_bytes=229384 combine_payload_bytes=377309184 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=377538568 ingress_bytes=597510144 link_bytes=597510144 token_bytes=754847752 weight_movement_bytes=220200960 total_nvlink_bytes=975048712 weight_to_token_ratio=0.291716 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.039168414e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64256 payload_rows_remote=56064 dup_rows=1280 dup_rows_remote=192 dedup_ratio=0.019531 router_tokens_per_expert=[0, 0, 0, 4072, 2, 0, 4095, 0, 0, 3908, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4094, 62, 0, 0, 0, 420, 4096, 0, 124, 0, 0, 0, 0, 0, 0, 0, 49, 3652, 2, 4077, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4090, 0, 0, 0, 0, 8] payload_rows_per_dst=[8144, 7816, 8188, 8192, 7388, 8158, 8190, 8180] dup_rows_per_dst=[4, 0, 134, 1088, 18, 2, 0, 34] combine_rows_per_src=[8144, 7816, 8188, 8192, 7388, 8158, 8190, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60034 payload_rows_remote=51846 dup_rows=5502 dup_rows_remote=4412 dedup_ratio=0.083954 router_tokens_per_expert=[0, 0, 0, 4094, 1, 0, 4096, 0, 0, 3935, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 4093, 111, 0, 0, 0, 477, 4096, 0, 237, 0, 0, 0, 0, 0, 0, 0, 51, 3372, 2, 4063, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4088, 0, 0, 0, 0, 37] payload_rows_per_dst=[8188, 8002, 7286, 5392, 6838, 8130, 8192, 8006] dup_rows_per_dst=[1090, 3298, 224, 800, 12, 4, 0, 74] combine_rows_per_src=[8188, 8002, 7286, 5392, 6838, 8130, 8192, 8006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=388950016 dispatch_meta_bytes=227840 combine_payload_bytes=388950016 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=389177856 ingress_bytes=829351936 link_bytes=829351936 token_bytes=778127872 weight_movement_bytes=440401920 total_nvlink_bytes=1218529792 weight_to_token_ratio=0.565976 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.442379422e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=401866752 dispatch_meta_bytes=225024 combine_payload_bytes=401866752 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=402091776 ingress_bytes=732168192 link_bytes=732168192 token_bytes=803958528 weight_movement_bytes=330301440 total_nvlink_bytes=1134259968 weight_to_token_ratio=0.410844 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=25088 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.273360907e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=371632128 dispatch_meta_bytes=225032 combine_payload_bytes=371632128 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=371857160 ingress_bytes=701933568 link_bytes=701933568 token_bytes=743489288 weight_movement_bytes=330301440 total_nvlink_bytes=1073790728 weight_to_token_ratio=0.444259 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.220777923e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63002 payload_rows_remote=54878 dup_rows=2534 dup_rows_remote=2530 dedup_ratio=0.038666 router_tokens_per_expert=[0, 0, 3007, 0, 0, 4096, 0, 0, 1, 0, 0, 72, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 4096, 0, 0, 4060, 0, 4018, 0, 2, 7, 0, 0, 0, 0, 0, 21, 600, 0, 4064, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 517, 4091, 0] payload_rows_per_dst=[6014, 8192, 8078, 8036, 8176, 8124, 8192, 8190] dup_rows_per_dst=[0, 146, 0, 18, 1194, 4, 0, 1172] combine_rows_per_src=[6014, 8192, 8078, 8036, 8176, 8124, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58658 payload_rows_remote=50484 dup_rows=6878 dup_rows_remote=6800 dedup_ratio=0.104950 router_tokens_per_expert=[0, 0, 3018, 0, 0, 4087, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 4096, 0, 0, 4063, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 20, 810, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 371, 4096, 0] payload_rows_per_dst=[8192, 8174, 8126, 8068, 2262, 8126, 8192, 7518] dup_rows_per_dst=[5906, 78, 0, 0, 82, 0, 0, 812] combine_rows_per_src=[8192, 8174, 8126, 8068, 2262, 8126, 8192, 7518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63210 payload_rows_remote=55018 dup_rows=2326 dup_rows_remote=1802 dedup_ratio=0.035492 router_tokens_per_expert=[0, 0, 3448, 0, 0, 4076, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 4096, 0, 0, 4043, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 36, 479, 0, 3954, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 218, 4096, 0] payload_rows_per_dst=[6896, 8152, 8082, 7498, 8088, 8110, 8192, 8192] dup_rows_per_dst=[0, 378, 0, 0, 850, 560, 14, 524] combine_rows_per_src=[6896, 8152, 8082, 7498, 8088, 8110, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62680 payload_rows_remote=54540 dup_rows=2856 dup_rows_remote=1114 dedup_ratio=0.043579 router_tokens_per_expert=[0, 0, 2786, 0, 0, 4093, 0, 0, 1, 0, 0, 70, 1, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4095, 0, 0, 4048, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 38, 866, 0, 4037, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 442, 4095, 0] payload_rows_per_dst=[5572, 8186, 8166, 8138, 8140, 8096, 8190, 8192] dup_rows_per_dst=[0, 144, 0, 0, 1742, 4, 0, 966] combine_rows_per_src=[5572, 8186, 8166, 8138, 8140, 8096, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61542 payload_rows_remote=53358 dup_rows=3994 dup_rows_remote=3994 dedup_ratio=0.060944 router_tokens_per_expert=[0, 0, 2182, 0, 0, 4094, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4092, 0, 0, 4065, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 20, 806, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 978, 4096, 0] payload_rows_per_dst=[4364, 8188, 8172, 8158, 8154, 8130, 8184, 8192] dup_rows_per_dst=[0, 432, 0, 0, 1546, 0, 0, 2016] combine_rows_per_src=[4364, 8188, 8172, 8158, 8154, 8130, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=393365504 dispatch_meta_bytes=229632 combine_payload_bytes=393365504 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=393595136 ingress_bytes=613566464 link_bytes=613566464 token_bytes=786960640 weight_movement_bytes=220200960 total_nvlink_bytes=1007161600 weight_to_token_ratio=0.279812 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.067092995e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=361869312 dispatch_meta_bytes=229136 combine_payload_bytes=361869312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362098448 ingress_bytes=471969792 link_bytes=471969792 token_bytes=723967760 weight_movement_bytes=110100480 total_nvlink_bytes=834068240 weight_to_token_ratio=0.152079 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.208330940e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61056 payload_rows_remote=53068 dup_rows=4480 dup_rows_remote=4480 dedup_ratio=0.068359 router_tokens_per_expert=[0, 0, 2796, 0, 0, 4096, 0, 0, 3, 0, 0, 275, 2, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 4095, 0, 0, 4068, 0, 4058, 0, 0, 2, 0, 0, 0, 0, 0, 30, 775, 0, 3984, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 444, 4093, 0] payload_rows_per_dst=[6136, 8192, 7988, 8116, 6102, 8140, 8190, 8192] dup_rows_per_dst=[1900, 560, 0, 4, 1032, 0, 0, 984] combine_rows_per_src=[6136, 8192, 7988, 8116, 6102, 8140, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=394369024 dispatch_meta_bytes=227280 combine_payload_bytes=394369024 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=394596304 ingress_bytes=724670464 link_bytes=724670464 token_bytes=788965328 weight_movement_bytes=330301440 total_nvlink_bytes=1119266768 weight_to_token_ratio=0.418651 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.260321125e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=390942720 dispatch_meta_bytes=222616 combine_payload_bytes=390942720 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=391165336 ingress_bytes=611143680 link_bytes=611143680 token_bytes=782108056 weight_movement_bytes=220200960 total_nvlink_bytes=1002309016 weight_to_token_ratio=0.281548 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062879375e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=382470144 dispatch_meta_bytes=229408 combine_payload_bytes=382470144 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=382699552 ingress_bytes=602671104 link_bytes=602671104 token_bytes=765169696 weight_movement_bytes=220200960 total_nvlink_bytes=985370656 weight_to_token_ratio=0.287781 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.048144172e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62334 payload_rows_remote=54238 dup_rows=3202 dup_rows_remote=3200 dedup_ratio=0.048859 router_tokens_per_expert=[0, 0, 2647, 0, 0, 4096, 0, 0, 3, 0, 0, 131, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 4095, 0, 0, 4047, 0, 4048, 0, 0, 1, 0, 0, 0, 0, 0, 38, 806, 0, 4067, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 586, 4094, 0] payload_rows_per_dst=[5294, 8192, 8118, 8096, 8158, 8094, 8190, 8192] dup_rows_per_dst=[0, 268, 0, 2, 1664, 8, 0, 1260] combine_rows_per_src=[5294, 8192, 8118, 8096, 8158, 8094, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54362 payload_rows_remote=46170 dup_rows=11174 dup_rows_remote=4104 dedup_ratio=0.170502 router_tokens_per_expert=[0, 0, 2428, 0, 0, 4091, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4096, 0, 0, 4068, 0, 4053, 0, 0, 3, 0, 0, 0, 0, 0, 25, 1037, 0, 3973, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 513, 4094, 0] payload_rows_per_dst=[8192, 5860, 8156, 8124, 6568, 8136, 8192, 1134] dup_rows_per_dst=[7070, 394, 650, 1278, 1778, 4, 0, 0] combine_rows_per_src=[8192, 5860, 8156, 8124, 6568, 8136, 8192, 1134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=380391424 dispatch_meta_bytes=230192 combine_payload_bytes=380391424 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=380621616 ingress_bytes=710692864 link_bytes=710692864 token_bytes=761013040 weight_movement_bytes=330301440 total_nvlink_bytes=1091314480 weight_to_token_ratio=0.434029 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.236011779e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61716 payload_rows_remote=53758 dup_rows=3820 dup_rows_remote=3814 dedup_ratio=0.058289 router_tokens_per_expert=[0, 42, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 834, 0, 0, 0, 0, 0, 0, 3769, 5, 0, 0, 0, 0, 4067, 4048, 0, 4080, 0, 0, 0, 0, 3, 0, 3979, 0, 0, 4044, 0, 0, 0, 40, 0, 0, 0, 2890, 36, 0, 0, 0, 1307, 0, 0, 0, 3573, 0] payload_rows_per_dst=[8190, 8160, 8104, 7548, 8096, 7958, 5838, 7822] dup_rows_per_dst=[130, 0, 1652, 0, 0, 6, 94, 1938] combine_rows_per_src=[8190, 8160, 8104, 7548, 8096, 7958, 5838, 7822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=388777984 dispatch_meta_bytes=229752 combine_payload_bytes=388777984 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=389007736 ingress_bytes=719079424 link_bytes=719079424 token_bytes=777785720 weight_movement_bytes=330301440 total_nvlink_bytes=1108087160 weight_to_token_ratio=0.424669 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250597387e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=330946560 dispatch_meta_bytes=201096 combine_payload_bytes=330946560 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=331147656 ingress_bytes=551147520 link_bytes=551147520 token_bytes=662094216 weight_movement_bytes=220200960 total_nvlink_bytes=882295176 weight_to_token_ratio=0.332583 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=22400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.585361855e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58154 payload_rows_remote=49962 dup_rows=7382 dup_rows_remote=7382 dedup_ratio=0.112640 router_tokens_per_expert=[0, 2, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 824, 0, 0, 0, 0, 0, 0, 3886, 1, 0, 0, 0, 0, 4086, 4069, 0, 4096, 0, 0, 0, 0, 1, 0, 4079, 0, 0, 4048, 0, 0, 0, 3, 0, 0, 0, 3095, 12, 0, 0, 0, 717, 0, 0, 0, 3787, 0] payload_rows_per_dst=[8176, 8192, 8114, 7774, 8138, 8158, 8168, 1434] dup_rows_per_dst=[124, 0, 1630, 0, 0, 2, 5626, 0] combine_rows_per_src=[8176, 8192, 8114, 7774, 8138, 8158, 8168, 1434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52912 payload_rows_remote=44720 dup_rows=12624 dup_rows_remote=1608 dedup_ratio=0.192627 router_tokens_per_expert=[0, 2, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 0, 0, 3659, 0, 0, 0, 0, 0, 4096, 4046, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4047, 0, 0, 0, 27, 0, 0, 0, 2445, 3, 0, 0, 0, 1995, 0, 0, 0, 3515, 0] payload_rows_per_dst=[6742, 8192, 1336, 7318, 8100, 8104, 4928, 8192] dup_rows_per_dst=[128, 0, 0, 0, 500, 958, 22, 11016] combine_rows_per_src=[6742, 8192, 1336, 7318, 8100, 8104, 4928, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62112 payload_rows_remote=53932 dup_rows=3424 dup_rows_remote=3424 dedup_ratio=0.052246 router_tokens_per_expert=[0, 1, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 875, 0, 0, 0, 0, 0, 0, 4006, 2, 0, 0, 0, 0, 4095, 4090, 0, 4095, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4053, 0, 0, 0, 1, 0, 0, 0, 2861, 29, 0, 0, 0, 986, 0, 0, 0, 3407, 0] payload_rows_per_dst=[8192, 8190, 8110, 8016, 8180, 8174, 5732, 7518] dup_rows_per_dst=[360, 0, 1746, 0, 0, 0, 50, 1268] combine_rows_per_src=[8192, 8190, 8110, 8016, 8180, 8174, 5732, 7518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58810 payload_rows_remote=53386 dup_rows=6726 dup_rows_remote=6692 dedup_ratio=0.102631 router_tokens_per_expert=[0, 4, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 0, 0, 0, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 4095, 4061, 0, 4096, 0, 0, 0, 0, 16, 0, 4036, 0, 0, 4052, 0, 0, 0, 30, 0, 0, 0, 2683, 16, 0, 0, 0, 739, 0, 0, 0, 3811, 0] payload_rows_per_dst=[8192, 8192, 4914, 7974, 8122, 8118, 5424, 7874] dup_rows_per_dst=[202, 0, 1274, 0, 0, 2890, 34, 2326] combine_rows_per_src=[8192, 8192, 4914, 7974, 8122, 8118, 5424, 7874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=385337344 dispatch_meta_bytes=230288 combine_payload_bytes=385337344 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=385567632 ingress_bytes=825739264 link_bytes=825739264 token_bytes=770904976 weight_movement_bytes=440401920 total_nvlink_bytes=1211306896 weight_to_token_ratio=0.571279 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=26240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.436096391e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=358127616 dispatch_meta_bytes=229376 combine_payload_bytes=358127616 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=358356992 ingress_bytes=578328576 link_bytes=578328576 token_bytes=716484608 weight_movement_bytes=220200960 total_nvlink_bytes=936685568 weight_to_token_ratio=0.307335 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.005808512e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=320552960 dispatch_meta_bytes=185312 combine_payload_bytes=320552960 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=320738272 ingress_bytes=540753920 link_bytes=540753920 token_bytes=641291232 weight_movement_bytes=220200960 total_nvlink_bytes=861492192 weight_to_token_ratio=0.343371 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=18560 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.404600056e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60826 payload_rows_remote=52764 dup_rows=4710 dup_rows_remote=3456 dedup_ratio=0.071869 router_tokens_per_expert=[0, 5, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 0, 0, 0, 0, 0, 0, 3980, 2, 0, 0, 0, 0, 4060, 4047, 0, 4094, 0, 0, 0, 0, 1, 0, 4088, 0, 0, 4019, 0, 0, 0, 39, 0, 0, 0, 2078, 8, 0, 0, 0, 1974, 0, 0, 0, 3628, 0] payload_rows_per_dst=[8128, 8188, 8062, 7964, 8094, 8176, 7706, 4508] dup_rows_per_dst=[214, 0, 1254, 0, 0, 2, 2974, 266] combine_rows_per_src=[8128, 8188, 8062, 7964, 8094, 8176, 7706, 4508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=386584576 dispatch_meta_bytes=229424 combine_payload_bytes=386584576 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=386814000 ingress_bytes=826986496 link_bytes=826986496 token_bytes=773398576 weight_movement_bytes=440401920 total_nvlink_bytes=1213800496 weight_to_token_ratio=0.569437 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.438265532e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=382670848 dispatch_meta_bytes=240312 combine_payload_bytes=382670848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382911160 ingress_bytes=492771328 link_bytes=492771328 token_bytes=765582008 weight_movement_bytes=110100480 total_nvlink_bytes=875682488 weight_to_token_ratio=0.143813 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=21888 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.570103863e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56074 payload_rows_remote=47894 dup_rows=9462 dup_rows_remote=9254 dedup_ratio=0.144379 router_tokens_per_expert=[0, 17, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 608, 0, 0, 0, 0, 0, 0, 3952, 3, 0, 0, 0, 0, 4076, 4086, 0, 4094, 0, 0, 0, 0, 2, 0, 4061, 0, 0, 4043, 0, 0, 0, 16, 0, 0, 0, 2961, 9, 0, 0, 0, 1060, 0, 0, 0, 3679, 0] payload_rows_per_dst=[8180, 8192, 8088, 8060, 8126, 5138, 8170, 2120] dup_rows_per_dst=[208, 42, 1214, 2838, 0, 0, 5160, 0] combine_rows_per_src=[8180, 8192, 8088, 8060, 8126, 5138, 8170, 2120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62036 payload_rows_remote=53992 dup_rows=3500 dup_rows_remote=3500 dedup_ratio=0.053406 router_tokens_per_expert=[0, 19, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 0, 0, 0, 0, 0, 0, 4021, 1, 0, 0, 0, 0, 4089, 4090, 0, 4094, 0, 0, 0, 0, 2, 0, 3924, 0, 0, 4044, 0, 0, 0, 1, 0, 0, 0, 2787, 17, 0, 0, 0, 793, 0, 0, 0, 3834, 0] payload_rows_per_dst=[8188, 8188, 8096, 8044, 8180, 7848, 5610, 7882] dup_rows_per_dst=[182, 0, 1942, 0, 0, 4, 0, 1372] combine_rows_per_src=[8188, 8188, 8096, 8044, 8180, 7848, 5610, 7882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=378212352 dispatch_meta_bytes=224880 combine_payload_bytes=378212352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=378437232 ingress_bytes=488312832 link_bytes=488312832 token_bytes=756649584 weight_movement_bytes=110100480 total_nvlink_bytes=866750064 weight_to_token_ratio=0.145511 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=26368 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.492563285e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=343304192 dispatch_meta_bytes=228592 combine_payload_bytes=343304192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=343532784 ingress_bytes=453404672 link_bytes=453404672 token_bytes=686836976 weight_movement_bytes=110100480 total_nvlink_bytes=796937456 weight_to_token_ratio=0.160301 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.885452969e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62398 payload_rows_remote=56864 dup_rows=3138 dup_rows_remote=2922 dedup_ratio=0.047882 router_tokens_per_expert=[0, 78, 0, 4094, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 500, 0, 41, 2334, 0, 0, 4096, 0, 0, 0, 1424, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8024, 8188, 8192, 7972, 8192, 5534, 8104, 8192] dup_rows_per_dst=[124, 0, 68, 0, 0, 216, 2730, 0] combine_rows_per_src=[8024, 8188, 8192, 7972, 8192, 5534, 8104, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59222 payload_rows_remote=51030 dup_rows=6314 dup_rows_remote=6314 dedup_ratio=0.096344 router_tokens_per_expert=[0, 46, 0, 4096, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 367, 0, 0, 2750, 0, 0, 4096, 0, 0, 0, 1035, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8154, 8192, 8192, 8040, 8192, 8190, 2070, 8192] dup_rows_per_dst=[84, 0, 16, 0, 0, 6214, 0, 0] combine_rows_per_src=[8154, 8192, 8192, 8040, 8192, 8190, 2070, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=387014656 dispatch_meta_bytes=229968 combine_payload_bytes=387014656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=387244624 ingress_bytes=497115136 link_bytes=497115136 token_bytes=774259280 weight_movement_bytes=110100480 total_nvlink_bytes=884359760 weight_to_token_ratio=0.142201 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.645649829e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61208 payload_rows_remote=53016 dup_rows=4328 dup_rows_remote=4328 dedup_ratio=0.066040 router_tokens_per_expert=[0, 254, 0, 4096, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 840, 0, 0, 1958, 0, 0, 4096, 0, 0, 0, 1262, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6862, 8192, 8192, 8016, 8192, 5414, 8148, 8192] dup_rows_per_dst=[480, 6, 10, 1148, 0, 182, 2502, 0] combine_rows_per_src=[6862, 8192, 8192, 8016, 8192, 5414, 8148, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62926 payload_rows_remote=54734 dup_rows=2610 dup_rows_remote=2610 dedup_ratio=0.039825 router_tokens_per_expert=[0, 46, 0, 4095, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 441, 0, 1, 2452, 0, 0, 4096, 0, 0, 0, 1236, 0, 0, 4082, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8176, 8192, 8192, 8078, 8192, 5722, 8182, 8192] dup_rows_per_dst=[88, 0, 0, 0, 0, 66, 2454, 2] combine_rows_per_src=[8176, 8192, 8192, 8078, 8192, 5722, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62014 payload_rows_remote=53830 dup_rows=3522 dup_rows_remote=256 dedup_ratio=0.053741 router_tokens_per_expert=[0, 134, 0, 4096, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 806, 0, 0, 1567, 0, 0, 4096, 0, 0, 0, 1646, 0, 0, 4079, 0, 0, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[8174, 8192, 8192, 8158, 8192, 4730, 8184, 8192] dup_rows_per_dst=[210, 0, 24, 0, 0, 16, 3266, 6] combine_rows_per_src=[8174, 8192, 8192, 8158, 8192, 4730, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=407601152 dispatch_meta_bytes=239144 combine_payload_bytes=407601152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407840296 ingress_bytes=517701632 link_bytes=517701632 token_bytes=815441448 weight_movement_bytes=110100480 total_nvlink_bytes=925541928 weight_to_token_ratio=0.135019 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.003682853e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=365783040 dispatch_meta_bytes=229376 combine_payload_bytes=365783040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366012416 ingress_bytes=475883520 link_bytes=475883520 token_bytes=731795456 weight_movement_bytes=110100480 total_nvlink_bytes=841895936 weight_to_token_ratio=0.150453 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.276397107e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=380018688 dispatch_meta_bytes=229376 combine_payload_bytes=380018688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380248064 ingress_bytes=490119168 link_bytes=490119168 token_bytes=760266752 weight_movement_bytes=110100480 total_nvlink_bytes=870367232 weight_to_token_ratio=0.144818 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.523978439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=392333312 dispatch_meta_bytes=229376 combine_payload_bytes=392333312 prefetch_bytes=0 reduce_bytes=0 egress_bytes=392562688 ingress_bytes=392333312 link_bytes=392562688 token_bytes=784896000 weight_movement_bytes=0 total_nvlink_bytes=784896000 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.827310799e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61006 payload_rows_remote=52816 dup_rows=4530 dup_rows_remote=4518 dedup_ratio=0.069122 router_tokens_per_expert=[0, 205, 0, 4092, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1313, 0, 5, 1953, 0, 0, 4096, 0, 0, 0, 1076, 0, 0, 3812, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8116, 8192, 8190, 7976, 8192, 6990, 5158, 8192] dup_rows_per_dst=[344, 0, 12, 0, 0, 3050, 1120, 4] combine_rows_per_src=[8116, 8192, 8190, 7976, 8192, 6990, 5158, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=385853440 dispatch_meta_bytes=216344 combine_payload_bytes=385853440 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=386069784 ingress_bytes=716154880 link_bytes=716154880 token_bytes=771923224 weight_movement_bytes=330301440 total_nvlink_bytes=1102224664 weight_to_token_ratio=0.427894 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=22272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.245511124e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59360 payload_rows_remote=51210 dup_rows=6176 dup_rows_remote=5792 dedup_ratio=0.094238 router_tokens_per_expert=[0, 125, 0, 4095, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 887, 0, 4, 2084, 0, 0, 4096, 0, 0, 0, 1150, 0, 0, 4013, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8150, 8190, 8032, 8118, 8192, 8188, 2312, 8178] dup_rows_per_dst=[384, 0, 0, 0, 0, 5788, 4, 0] combine_rows_per_src=[8150, 8190, 8032, 8118, 8192, 8188, 2312, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62064 payload_rows_remote=54034 dup_rows=3472 dup_rows_remote=3472 dedup_ratio=0.052979 router_tokens_per_expert=[0, 67, 0, 4095, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 385, 0, 9, 2224, 0, 0, 4096, 0, 0, 0, 1739, 0, 0, 3912, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8048, 8192, 8192, 8030, 8192, 5088, 8130, 8192] dup_rows_per_dst=[118, 0, 32, 0, 0, 148, 3172, 2] combine_rows_per_src=[8048, 8192, 8192, 8030, 8192, 5088, 8130, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=378585088 dispatch_meta_bytes=229336 combine_payload_bytes=378585088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=378814424 ingress_bytes=488685568 link_bytes=488685568 token_bytes=757399512 weight_movement_bytes=110100480 total_nvlink_bytes=867499992 weight_to_token_ratio=0.145366 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.499045777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63352 payload_rows_remote=55556 dup_rows=2184 dup_rows_remote=2184 dedup_ratio=0.033325 router_tokens_per_expert=[0, 4081, 0, 0, 0, 3370, 0, 0, 4026, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 0, 0, 0, 0, 4096, 0, 4094, 3936, 0, 0, 0, 0, 4096, 57, 0, 11, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6740, 8066, 8192, 8192, 8192, 7796, 7986, 8188] dup_rows_per_dst=[0, 0, 12, 2150, 0, 0, 0, 22] combine_rows_per_src=[6740, 8066, 8192, 8192, 8192, 7796, 7986, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61410 payload_rows_remote=53244 dup_rows=4126 dup_rows_remote=4126 dedup_ratio=0.062958 router_tokens_per_expert=[0, 4095, 0, 0, 0, 3755, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 0, 0, 4096, 0, 4092, 3997, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7776, 8166, 8192, 5024, 8192, 7862, 8014, 8184] dup_rows_per_dst=[3656, 0, 16, 454, 0, 0, 0, 0] combine_rows_per_src=[7776, 8166, 8192, 5024, 8192, 7862, 8014, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=367073280 dispatch_meta_bytes=228008 combine_payload_bytes=367073280 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=367301288 ingress_bytes=807475200 link_bytes=807475200 token_bytes=734374568 weight_movement_bytes=440401920 total_nvlink_bytes=1174776488 weight_to_token_ratio=0.599697 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.404332180e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=387315712 dispatch_meta_bytes=230024 combine_payload_bytes=387315712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=387545736 ingress_bytes=497416192 link_bytes=497416192 token_bytes=774861448 weight_movement_bytes=110100480 total_nvlink_bytes=884961928 weight_to_token_ratio=0.142091 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.650885688e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58648 payload_rows_remote=50456 dup_rows=6888 dup_rows_remote=6858 dedup_ratio=0.105103 router_tokens_per_expert=[0, 4096, 0, 0, 0, 3626, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3643, 0, 0, 0, 0, 4096, 0, 4096, 3902, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2658, 8166, 8090, 8192, 8192, 7350, 7808, 8192] dup_rows_per_dst=[0, 280, 26, 2232, 0, 3284, 1036, 30] combine_rows_per_src=[2658, 8166, 8090, 8192, 8192, 7350, 7808, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64254 payload_rows_remote=56062 dup_rows=1282 dup_rows_remote=1282 dedup_ratio=0.019562 router_tokens_per_expert=[0, 4095, 0, 0, 0, 3557, 0, 0, 4077, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 4096, 0, 4096, 4075, 0, 0, 0, 0, 4096, 1, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7114, 8156, 8192, 8192, 8192, 8064, 8152, 8192] dup_rows_per_dst=[0, 0, 2, 1278, 0, 0, 0, 2] combine_rows_per_src=[7114, 8156, 8192, 8192, 8192, 8064, 8152, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64896 payload_rows_remote=56720 dup_rows=640 dup_rows_remote=640 dedup_ratio=0.009766 router_tokens_per_expert=[0, 4096, 0, 0, 0, 3916, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 0, 0, 0, 0, 4096, 0, 4090, 4069, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7832, 8192, 8192, 8192, 8192, 7940, 8176, 8180] dup_rows_per_dst=[0, 0, 8, 632, 0, 0, 0, 0] combine_rows_per_src=[7832, 8192, 8192, 8192, 8192, 7940, 8176, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=398225408 dispatch_meta_bytes=230960 combine_payload_bytes=398225408 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=398456368 ingress_bytes=618426368 link_bytes=618426368 token_bytes=796681776 weight_movement_bytes=220200960 total_nvlink_bytes=1016882736 weight_to_token_ratio=0.276398 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.075545168e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=381652992 dispatch_meta_bytes=229480 combine_payload_bytes=381652992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=381882472 ingress_bytes=491753472 link_bytes=491753472 token_bytes=763535464 weight_movement_bytes=110100480 total_nvlink_bytes=873635944 weight_to_token_ratio=0.144198 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.552401674e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=361668608 dispatch_meta_bytes=229256 combine_payload_bytes=361668608 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=361897864 ingress_bytes=581869568 link_bytes=581869568 token_bytes=723566472 weight_movement_bytes=220200960 total_nvlink_bytes=943767432 weight_to_token_ratio=0.304327 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.011966880e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=401852416 dispatch_meta_bytes=229376 combine_payload_bytes=401852416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402081792 ingress_bytes=511952896 link_bytes=511952896 token_bytes=803934208 weight_movement_bytes=110100480 total_nvlink_bytes=914034688 weight_to_token_ratio=0.136952 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.903702879e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=406568960 dispatch_meta_bytes=229440 combine_payload_bytes=406568960 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=406798400 ingress_bytes=846970880 link_bytes=846970880 token_bytes=813367360 weight_movement_bytes=440401920 total_nvlink_bytes=1253769280 weight_to_token_ratio=0.541455 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.473021663e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63356 payload_rows_remote=55164 dup_rows=2180 dup_rows_remote=2178 dedup_ratio=0.033264 router_tokens_per_expert=[0, 4093, 0, 0, 0, 3497, 0, 0, 4071, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3681, 0, 0, 0, 0, 4096, 0, 4093, 4008, 0, 0, 0, 0, 4096, 38, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6994, 8146, 8192, 8192, 8192, 7362, 8092, 8186] dup_rows_per_dst=[0, 2, 2, 2172, 0, 0, 0, 4] combine_rows_per_src=[6994, 8146, 8192, 8192, 8192, 7362, 8092, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58516 payload_rows_remote=50324 dup_rows=7020 dup_rows_remote=36 dedup_ratio=0.107117 router_tokens_per_expert=[0, 4094, 0, 0, 0, 3494, 0, 0, 4086, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 2, 0, 0, 0, 0, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 0, 0, 0, 0, 4096, 0, 4090, 4024, 0, 0, 0, 0, 4096, 32, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8174, 8192, 1600, 8192, 7874, 8112, 8180] dup_rows_per_dst=[6984, 0, 30, 4, 0, 0, 0, 2] combine_rows_per_src=[8192, 8174, 8192, 1600, 8192, 7874, 8112, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64688 payload_rows_remote=56496 dup_rows=848 dup_rows_remote=10 dedup_ratio=0.012939 router_tokens_per_expert=[0, 4094, 0, 0, 0, 3795, 0, 0, 4092, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 4096, 0, 4085, 4062, 0, 0, 0, 0, 4096, 18, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7590, 8188, 8192, 8192, 8192, 8004, 8160, 8170] dup_rows_per_dst=[0, 0, 6, 838, 0, 0, 0, 4] combine_rows_per_src=[7590, 8188, 8192, 8192, 8192, 8004, 8160, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=395415552 dispatch_meta_bytes=229368 combine_payload_bytes=395415552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=395644920 ingress_bytes=505516032 link_bytes=505516032 token_bytes=791060472 weight_movement_bytes=110100480 total_nvlink_bytes=901160952 weight_to_token_ratio=0.139181 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.791755227e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62064 payload_rows_remote=53872 dup_rows=3472 dup_rows_remote=3472 dedup_ratio=0.052979 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2606, 0, 0, 23, 0, 4096, 1579, 4068, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 138, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 0, 0, 0, 0, 0, 1, 0, 0, 0, 10, 0, 0, 0, 0, 0] payload_rows_per_dst=[5212, 8192, 8192, 8192, 8192, 8192, 7700, 8192] dup_rows_per_dst=[0, 3148, 22, 276, 4, 0, 2, 20] combine_rows_per_src=[5212, 8192, 8192, 8192, 8192, 8192, 7700, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59926 payload_rows_remote=57192 dup_rows=5610 dup_rows_remote=5610 dedup_ratio=0.085602 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2652, 0, 0, 0, 0, 4096, 1367, 4096, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 2734, 8192, 8192, 8192, 8192, 8040, 8192] dup_rows_per_dst=[5304, 0, 30, 274, 0, 0, 0, 2] combine_rows_per_src=[8192, 2734, 8192, 8192, 8192, 8192, 8040, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59214 payload_rows_remote=51022 dup_rows=6322 dup_rows_remote=6314 dedup_ratio=0.096466 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2215, 0, 0, 0, 0, 4096, 1729, 4092, 0, 0, 0, 34, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 223, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0] payload_rows_per_dst=[4430, 8192, 8006, 5966, 8192, 8192, 8044, 8192] dup_rows_per_dst=[0, 3450, 68, 290, 62, 0, 2444, 8] combine_rows_per_src=[4430, 8192, 8006, 5966, 8192, 8192, 8044, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62368 payload_rows_remote=54176 dup_rows=3168 dup_rows_remote=3168 dedup_ratio=0.048340 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2643, 0, 0, 1, 0, 4096, 1434, 4095, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5286, 8192, 8192, 8192, 8192, 8192, 7930, 8192] dup_rows_per_dst=[0, 2868, 16, 284, 0, 0, 0, 0] combine_rows_per_src=[5286, 8192, 8192, 8192, 8192, 8192, 7930, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=360722432 dispatch_meta_bytes=201440 combine_payload_bytes=360722432 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=360923872 ingress_bytes=801124352 link_bytes=801124352 token_bytes=721646304 weight_movement_bytes=440401920 total_nvlink_bytes=1162048224 weight_to_token_ratio=0.610274 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.393287010e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=404963328 dispatch_meta_bytes=226024 combine_payload_bytes=404963328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405189352 ingress_bytes=515063808 link_bytes=515063808 token_bytes=810152680 weight_movement_bytes=110100480 total_nvlink_bytes=920253160 weight_to_token_ratio=0.135901 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.957806756e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63394 payload_rows_remote=55584 dup_rows=2142 dup_rows_remote=2142 dedup_ratio=0.032684 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3216, 0, 0, 2, 0, 4096, 953, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 110, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6432, 8192, 8192, 8192, 8192, 8192, 7810, 8192] dup_rows_per_dst=[0, 1910, 2, 220, 10, 0, 0, 0] combine_rows_per_src=[6432, 8192, 8192, 8192, 8192, 8192, 7810, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=386154496 dispatch_meta_bytes=229376 combine_payload_bytes=386154496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=386383872 ingress_bytes=496254976 link_bytes=496254976 token_bytes=772538368 weight_movement_bytes=110100480 total_nvlink_bytes=882638848 weight_to_token_ratio=0.142518 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.630690232e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=409952256 dispatch_meta_bytes=251208 combine_payload_bytes=409952256 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=410203464 ingress_bytes=740253696 link_bytes=740253696 token_bytes=820155720 weight_movement_bytes=330301440 total_nvlink_bytes=1150457160 weight_to_token_ratio=0.402730 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=21504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.287422928e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=365725696 dispatch_meta_bytes=229344 combine_payload_bytes=365725696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=365955040 ingress_bytes=475826176 link_bytes=475826176 token_bytes=731680736 weight_movement_bytes=110100480 total_nvlink_bytes=841781216 weight_to_token_ratio=0.150476 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.275399800e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=388333568 dispatch_meta_bytes=229376 combine_payload_bytes=388333568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=388562944 ingress_bytes=498434048 link_bytes=498434048 token_bytes=776896512 weight_movement_bytes=110100480 total_nvlink_bytes=886996992 weight_to_token_ratio=0.141718 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.668587878e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=398426112 dispatch_meta_bytes=230904 combine_payload_bytes=398426112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398657016 ingress_bytes=508526592 link_bytes=508526592 token_bytes=797083128 weight_movement_bytes=110100480 total_nvlink_bytes=907183608 weight_to_token_ratio=0.138129 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.844113817e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60816 payload_rows_remote=52624 dup_rows=4720 dup_rows_remote=4692 dedup_ratio=0.072021 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 2417, 0, 0, 2, 0, 4096, 1402, 4093, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 439, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0] payload_rows_per_dst=[7642, 4418, 8192, 8190, 8192, 8192, 7798, 8192] dup_rows_per_dst=[3272, 500, 28, 878, 34, 0, 0, 8] combine_rows_per_src=[7642, 4418, 8192, 8190, 8192, 8192, 7798, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58148 payload_rows_remote=49956 dup_rows=7388 dup_rows_remote=790 dedup_ratio=0.112732 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 3299, 0, 0, 2, 0, 4096, 629, 4094, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 240, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 1, 0, 0, 0, 14, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 1944, 8192, 7414, 8066, 8190, 7996, 8154] dup_rows_per_dst=[6598, 382, 10, 388, 6, 2, 2, 0] combine_rows_per_src=[8192, 1944, 8192, 7414, 8066, 8190, 7996, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61892 payload_rows_remote=53702 dup_rows=3644 dup_rows_remote=3246 dedup_ratio=0.055603 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2488, 0, 0, 3, 0, 4096, 1610, 4094, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 199, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4976, 8192, 8192, 8190, 8192, 8192, 7766, 8192] dup_rows_per_dst=[0, 3222, 10, 398, 10, 0, 4, 0] combine_rows_per_src=[4976, 8192, 8192, 8190, 8192, 8192, 7766, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=377208832 dispatch_meta_bytes=229264 combine_payload_bytes=377208832 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=377438096 ingress_bytes=487309312 link_bytes=487309312 token_bytes=754646928 weight_movement_bytes=110100480 total_nvlink_bytes=864747408 weight_to_token_ratio=0.145897 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.475110422e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=358084608 dispatch_meta_bytes=202984 combine_payload_bytes=358084608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=358287592 ingress_bytes=468185088 link_bytes=468185088 token_bytes=716372200 weight_movement_bytes=110100480 total_nvlink_bytes=826472680 weight_to_token_ratio=0.153692 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=21632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.142508712e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61508 payload_rows_remote=53330 dup_rows=4028 dup_rows_remote=4020 dedup_ratio=0.061462 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 2569, 0, 0, 0, 0, 0, 0, 4096, 20, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 28, 162, 1565, 0, 0, 4089, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3596, 0, 0, 0, 0, 3998, 310, 0] payload_rows_per_dst=[5138, 8192, 8192, 8058, 8192, 8178, 8192, 7366] dup_rows_per_dst=[0, 40, 0, 24, 3510, 8, 0, 446] combine_rows_per_src=[5138, 8192, 8192, 8058, 8192, 8178, 8192, 7366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60782 payload_rows_remote=52590 dup_rows=4754 dup_rows_remote=4754 dedup_ratio=0.072540 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 2050, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 2, 167, 2284, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3362, 0, 0, 0, 0, 3884, 515, 0] payload_rows_per_dst=[8192, 8192, 8192, 7842, 4896, 8162, 8192, 7114] dup_rows_per_dst=[4100, 0, 0, 4, 10, 0, 0, 640] combine_rows_per_src=[8192, 8192, 8192, 7842, 4896, 8162, 8192, 7114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60536 payload_rows_remote=52344 dup_rows=5000 dup_rows_remote=274 dedup_ratio=0.076294 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 2490, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 1, 153, 2210, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3196, 0, 0, 0, 0, 3993, 272, 0] payload_rows_per_dst=[4980, 8182, 8192, 8058, 8192, 8066, 8192, 6674] dup_rows_per_dst=[0, 2, 2, 8, 4726, 0, 0, 262] combine_rows_per_src=[4980, 8182, 8192, 8058, 8192, 8066, 8192, 6674] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56244 payload_rows_remote=48058 dup_rows=9292 dup_rows_remote=3386 dedup_ratio=0.141785 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 3127, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 257, 1275, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3656, 0, 0, 0, 0, 3687, 263, 0] payload_rows_per_dst=[0, 8190, 8192, 7460, 7866, 8158, 8192, 8186] dup_rows_per_dst=[0, 0, 0, 0, 2752, 634, 0, 5906] combine_rows_per_src=[0, 8190, 8192, 7460, 7866, 8158, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=384935936 dispatch_meta_bytes=227792 combine_payload_bytes=384935936 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=385163728 ingress_bytes=935438336 link_bytes=935438336 token_bytes=770099664 weight_movement_bytes=550502400 total_nvlink_bytes=1320602064 weight_to_token_ratio=0.714846 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.626881119e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58296 payload_rows_remote=50104 dup_rows=7240 dup_rows_remote=7240 dedup_ratio=0.110474 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 2806, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 139, 2398, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2451, 0, 0, 0, 0, 4030, 461, 0] payload_rows_per_dst=[1642, 8192, 8192, 8132, 8192, 8114, 8192, 7640] dup_rows_per_dst=[0, 0, 0, 12, 5074, 0, 0, 2154] combine_rows_per_src=[1642, 8192, 8192, 8132, 8192, 8114, 8192, 7640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=382269440 dispatch_meta_bytes=229400 combine_payload_bytes=382269440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382498840 ingress_bytes=492369920 link_bytes=492369920 token_bytes=764768280 weight_movement_bytes=110100480 total_nvlink_bytes=874868760 weight_to_token_ratio=0.143966 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.563122718e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=376965120 dispatch_meta_bytes=229376 combine_payload_bytes=376965120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=377194496 ingress_bytes=597166080 link_bytes=597166080 token_bytes=754159616 weight_movement_bytes=220200960 total_nvlink_bytes=974360576 weight_to_token_ratio=0.291982 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.038570030e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=375201792 dispatch_meta_bytes=210472 combine_payload_bytes=375201792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375412264 ingress_bytes=485302272 link_bytes=485302272 token_bytes=750614056 weight_movement_bytes=110100480 total_nvlink_bytes=860714536 weight_to_token_ratio=0.146681 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.440204695e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=344479744 dispatch_meta_bytes=205776 combine_payload_bytes=344479744 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=344685520 ingress_bytes=674781184 link_bytes=674781184 token_bytes=689165264 weight_movement_bytes=330301440 total_nvlink_bytes=1019466704 weight_to_token_ratio=0.479278 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.173555461e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=359145472 dispatch_meta_bytes=229376 combine_payload_bytes=359145472 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=359374848 ingress_bytes=469245952 link_bytes=469245952 token_bytes=718520320 weight_movement_bytes=110100480 total_nvlink_bytes=828620800 weight_to_token_ratio=0.153232 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.160958882e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59674 payload_rows_remote=51482 dup_rows=5862 dup_rows_remote=5858 dedup_ratio=0.089447 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 2653, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 4, 296, 1364, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3632, 0, 0, 0, 0, 3958, 377, 0] payload_rows_per_dst=[8192, 8192, 8192, 8008, 3268, 8100, 8192, 7530] dup_rows_per_dst=[5302, 6, 4, 2, 60, 0, 0, 488] combine_rows_per_src=[8192, 8192, 8192, 8008, 3268, 8100, 8192, 7530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57896 payload_rows_remote=49704 dup_rows=7640 dup_rows_remote=1940 dedup_ratio=0.116577 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 2825, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 2, 165, 1847, 0, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 3243, 0, 0, 0, 0, 3974, 286, 0] payload_rows_per_dst=[8192, 8148, 8186, 8110, 4008, 7978, 8192, 5082] dup_rows_per_dst=[5700, 4, 0, 1702, 20, 0, 0, 214] combine_rows_per_src=[8192, 8148, 8186, 8110, 4008, 7978, 8192, 5082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59116 payload_rows_remote=51022 dup_rows=6420 dup_rows_remote=6398 dedup_ratio=0.097961 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 3060, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 2, 198, 2078, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2412, 0, 0, 0, 0, 4013, 505, 0] payload_rows_per_dst=[8148, 8190, 8192, 8094, 4614, 8138, 8192, 5548] dup_rows_per_dst=[5950, 6, 0, 22, 156, 0, 0, 286] combine_rows_per_src=[8148, 8190, 8192, 8094, 4614, 8138, 8192, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=369022976 dispatch_meta_bytes=229360 combine_payload_bytes=369022976 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=369252336 ingress_bytes=589223936 link_bytes=589223936 token_bytes=738275312 weight_movement_bytes=220200960 total_nvlink_bytes=958476272 weight_to_token_ratio=0.298264 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.024757335e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=356278272 dispatch_meta_bytes=206576 combine_payload_bytes=356278272 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=356484848 ingress_bytes=796680192 link_bytes=796680192 token_bytes=712763120 weight_movement_bytes=440401920 total_nvlink_bytes=1153165040 weight_to_token_ratio=0.617880 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.385557885e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60452 payload_rows_remote=52260 dup_rows=5084 dup_rows_remote=5084 dedup_ratio=0.077576 router_tokens_per_expert=[8, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 8, 4096, 0, 0, 0, 1772, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 759, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1575, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8160, 8192, 8192, 8192, 8182, 3150] dup_rows_per_dst=[6, 0, 0, 3560, 0, 1518, 0, 0] combine_rows_per_src=[8192, 8192, 8160, 8192, 8192, 8192, 8182, 3150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61370 payload_rows_remote=53188 dup_rows=4166 dup_rows_remote=3144 dedup_ratio=0.063568 router_tokens_per_expert=[53, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 7, 0, 4060, 0, 0, 0, 0, 0, 0, 3, 4067, 0, 0, 78, 4089, 0, 0, 0, 2076, 0, 0, 0, 0, 20, 0, 0, 0, 4093, 0, 0, 0, 4091, 0, 0, 511, 0, 0, 9, 0, 0, 0, 0, 4047, 0, 0, 1499, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8134, 8140, 4226, 8192, 8182, 8112, 8192] dup_rows_per_dst=[44, 0, 0, 82, 34, 1022, 0, 2984] combine_rows_per_src=[8192, 8134, 8140, 4226, 8192, 8182, 8112, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61764 payload_rows_remote=53572 dup_rows=3772 dup_rows_remote=3740 dedup_ratio=0.057556 router_tokens_per_expert=[2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 3, 4095, 0, 0, 0, 2215, 0, 0, 0, 0, 17, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 262, 0, 0, 1, 0, 0, 0, 0, 4092, 0, 0, 1605, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 4430, 8192, 8190, 8186, 8192] dup_rows_per_dst=[2, 0, 0, 6, 32, 524, 0, 3208] combine_rows_per_src=[8192, 8192, 8190, 4430, 8192, 8190, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59894 payload_rows_remote=51702 dup_rows=5642 dup_rows_remote=896 dedup_ratio=0.086090 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 8, 4096, 0, 0, 0, 1473, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 2373, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8058, 8192, 8090, 2962, 8022, 8192, 8186, 8192] dup_rows_per_dst=[0, 64, 0, 0, 4, 658, 170, 4746] combine_rows_per_src=[8058, 8192, 8090, 2962, 8022, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60824 payload_rows_remote=52656 dup_rows=4712 dup_rows_remote=4712 dedup_ratio=0.071899 router_tokens_per_expert=[4, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 11, 4096, 0, 0, 0, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 898, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 1446, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8188, 8130, 3578, 8192, 8192, 8168, 8192] dup_rows_per_dst=[4, 0, 0, 20, 0, 1796, 0, 2892] combine_rows_per_src=[8184, 8188, 8130, 3578, 8192, 8192, 8168, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=365725696 dispatch_meta_bytes=229680 combine_payload_bytes=365725696 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=365955376 ingress_bytes=585926656 link_bytes=585926656 token_bytes=731681072 weight_movement_bytes=220200960 total_nvlink_bytes=951882032 weight_to_token_ratio=0.300952 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.019022823e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=374599680 dispatch_meta_bytes=229376 combine_payload_bytes=374599680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374829056 ingress_bytes=484700160 link_bytes=484700160 token_bytes=749428736 weight_movement_bytes=110100480 total_nvlink_bytes=859529216 weight_to_token_ratio=0.146913 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.429732977e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=381251584 dispatch_meta_bytes=225328 combine_payload_bytes=381251584 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=381476912 ingress_bytes=711553024 link_bytes=711553024 token_bytes=762728496 weight_movement_bytes=330301440 total_nvlink_bytes=1093029936 weight_to_token_ratio=0.433052 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.237507739e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=384004096 dispatch_meta_bytes=229248 combine_payload_bytes=384004096 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=384233344 ingress_bytes=604205056 link_bytes=604205056 token_bytes=768237440 weight_movement_bytes=220200960 total_nvlink_bytes=988438400 weight_to_token_ratio=0.286631 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.050811967e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=370599936 dispatch_meta_bytes=210392 combine_payload_bytes=370599936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370810328 ingress_bytes=480700416 link_bytes=480700416 token_bytes=741410264 weight_movement_bytes=110100480 total_nvlink_bytes=851510744 weight_to_token_ratio=0.148501 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.360170851e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61396 payload_rows_remote=53260 dup_rows=4140 dup_rows_remote=4140 dedup_ratio=0.063171 router_tokens_per_expert=[20, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 1, 4067, 0, 0, 16, 4094, 0, 0, 0, 1664, 0, 0, 0, 0, 13, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 362, 0, 0, 2, 0, 1, 0, 0, 4091, 0, 0, 2060, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8136, 8192, 8190, 8188, 8186, 4120] dup_rows_per_dst=[34, 0, 0, 3356, 24, 724, 2, 0] combine_rows_per_src=[8192, 8192, 8136, 8192, 8190, 8188, 8186, 4120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=377438208 dispatch_meta_bytes=229472 combine_payload_bytes=377438208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=377667680 ingress_bytes=487538688 link_bytes=487538688 token_bytes=755105888 weight_movement_bytes=110100480 total_nvlink_bytes=865206368 weight_to_token_ratio=0.145808 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.479099648e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53188 payload_rows_remote=44996 dup_rows=12348 dup_rows_remote=12306 dedup_ratio=0.188416 router_tokens_per_expert=[24, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 4070, 0, 0, 13, 4094, 0, 0, 0, 2100, 0, 0, 0, 0, 32, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 0, 519, 0, 0, 2, 0, 0, 0, 0, 4089, 0, 0, 1450, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8184, 1154, 8182, 2900] dup_rows_per_dst=[42, 0, 342, 11896, 60, 8, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8184, 1154, 8182, 2900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7289883840 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=3646088384 ingress_bytes=5515503616 link_bytes=5515503616 total_nvlink_bytes=9161592000 weight_to_token_ratio=0.256754 prefetch_share_of_ingress=0.199620 dedup_ratio=0.112054 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6350 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.592367933e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7518702832 prefetch_bytes=1211105280 reduce_bytes=660602880 weight_movement_bytes=1871708160 egress_bytes=3760506096 ingress_bytes=5629904896 link_bytes=5629904896 total_nvlink_bytes=9390410992 weight_to_token_ratio=0.248940 prefetch_share_of_ingress=0.215120 dedup_ratio=0.086008 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=6 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.791330574e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7539743408 prefetch_bytes=880803840 reduce_bytes=770703360 weight_movement_bytes=1651507200 egress_bytes=3771009712 ingress_bytes=5420240896 link_bytes=5420470272 total_nvlink_bytes=9191250608 weight_to_token_ratio=0.219040 prefetch_share_of_ingress=0.162503 dedup_ratio=0.079675 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=7 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050735 gemm_inner_pad_upper_bound=5969 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.427089317e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6981336560 prefetch_bytes=880803840 reduce_bytes=1211105280 weight_movement_bytes=2091909120 egress_bytes=3491767792 ingress_bytes=5581477888 link_bytes=5581477888 total_nvlink_bytes=9073245680 weight_to_token_ratio=0.299643 prefetch_share_of_ingress=0.157808 dedup_ratio=0.138034 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=11 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6350 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.707108043e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7309170880 prefetch_bytes=1211105280 reduce_bytes=660602880 weight_movement_bytes=1871708160 egress_bytes=3655727296 ingress_bytes=5525151744 link_bytes=5525151744 total_nvlink_bytes=9180879040 weight_to_token_ratio=0.256077 prefetch_share_of_ingress=0.219199 dedup_ratio=0.105460 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=6 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048897 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.609147614e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=381767680 dispatch_meta_bytes=229600 combine_payload_bytes=381767680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=381997280 ingress_bytes=491868160 link_bytes=491868160 token_bytes=763764960 weight_movement_bytes=110100480 total_nvlink_bytes=873865440 weight_to_token_ratio=0.144155 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.554396287e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=322531328 dispatch_meta_bytes=229208 combine_payload_bytes=322531328 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=322760536 ingress_bytes=652832768 link_bytes=652832768 token_bytes=645291864 weight_movement_bytes=330301440 total_nvlink_bytes=975593304 weight_to_token_ratio=0.511864 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.135383556e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61930 payload_rows_remote=56104 dup_rows=3606 dup_rows_remote=2588 dedup_ratio=0.055023 router_tokens_per_expert=[18, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 16, 4096, 0, 0, 0, 2242, 0, 0, 0, 0, 15, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 404, 0, 0, 1, 0, 0, 0, 0, 4080, 0, 0, 1452, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8156, 5826, 8192, 8188, 8162, 7024] dup_rows_per_dst=[10, 0, 0, 1018, 26, 808, 0, 1744] combine_rows_per_src=[8192, 8190, 8156, 5826, 8192, 8188, 8162, 7024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7362856640 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=3682576064 ingress_bytes=5441888256 link_bytes=5441888256 total_nvlink_bytes=9124464320 weight_to_token_ratio=0.239256 prefetch_share_of_ingress=0.182088 dedup_ratio=0.098611 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050735 gemm_inner_pad_upper_bound=6096 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.464338714e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6792599248 prefetch_bytes=880803840 reduce_bytes=2422210560 weight_movement_bytes=3303014400 egress_bytes=3397375696 ingress_bytes=6698237952 link_bytes=6698237952 total_nvlink_bytes=10095613648 weight_to_token_ratio=0.486267 prefetch_share_of_ingress=0.131498 dedup_ratio=0.154773 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=22 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051471 gemm_inner_pad_upper_bound=5842 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.164933747e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=402153472 dispatch_meta_bytes=234768 combine_payload_bytes=402153472 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=402388240 ingress_bytes=622354432 link_bytes=622354432 token_bytes=804541712 weight_movement_bytes=220200960 total_nvlink_bytes=1024742672 weight_to_token_ratio=0.273697 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.082376717e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7556944040 prefetch_bytes=1211105280 reduce_bytes=1101004800 weight_movement_bytes=2312110080 egress_bytes=3779623080 ingress_bytes=6089431040 link_bytes=6089431040 total_nvlink_bytes=9869054120 weight_to_token_ratio=0.305958 prefetch_share_of_ingress=0.198886 dedup_ratio=0.084491 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=10 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6858 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.059052212e-04 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none + [2026-08-26 18:09:53] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 365.7 | throughput per GPU (TFLOP/s/GPU): 337.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.671047E+00 | loss scale: 1.0 | grad norm: 4.294 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44514 payload_rows_remote=38460 dup_rows=21022 dup_rows_remote=17364 dedup_ratio=0.320770 router_tokens_per_expert=[538, 1141, 970, 552, 80, 1087, 501, 40, 824, 1398, 1044, 306, 9, 1271, 69, 1155, 1005, 838, 20, 37, 434, 211, 533, 573, 252, 469, 585, 802, 323, 257, 347, 10, 839, 759, 806, 15, 891, 0, 951, 371, 584, 301, 584, 793, 4, 786, 142, 391, 219, 148, 98, 484, 112, 1143, 136, 640, 1131, 800, 795, 205, 327, 377, 253, 2] payload_rows_per_dst=[4956, 5182, 5404, 5516, 5770, 6054, 5116, 6516] dup_rows_per_dst=[1612, 1632, 1898, 2748, 3494, 3658, 1920, 4060] combine_rows_per_src=[4956, 5182, 5404, 5516, 5770, 6054, 5116, 6516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=39338 dup_rows=21110 dup_rows_remote=18594 dedup_ratio=0.322113 router_tokens_per_expert=[1187, 977, 1086, 872, 659, 1422, 741, 32, 199, 1369, 861, 465, 5, 878, 86, 470, 843, 615, 7, 29, 761, 395, 909, 497, 257, 810, 524, 618, 228, 182, 819, 2, 609, 240, 590, 8, 953, 2, 1013, 387, 262, 725, 417, 918, 0, 625, 152, 384, 219, 159, 312, 445, 259, 1262, 50, 502, 623, 935, 426, 90, 407, 797, 188, 4] payload_rows_per_dst=[5868, 4226, 5692, 6288, 5088, 5316, 5988, 5960] dup_rows_per_dst=[2866, 924, 2420, 3436, 2516, 2428, 2802, 3718] combine_rows_per_src=[5868, 4226, 5692, 6288, 5088, 5316, 5988, 5960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44652 payload_rows_remote=38826 dup_rows=20884 dup_rows_remote=17934 dedup_ratio=0.318665 router_tokens_per_expert=[1518, 431, 475, 948, 347, 452, 584, 198, 401, 1082, 960, 197, 9, 1335, 10, 976, 498, 318, 29, 63, 672, 396, 453, 620, 230, 1033, 342, 1060, 237, 268, 279, 6, 598, 318, 1022, 2, 881, 25, 1402, 645, 287, 663, 321, 1072, 5, 817, 106, 504, 183, 219, 403, 491, 661, 976, 165, 1269, 525, 887, 545, 218, 261, 510, 360, 0] payload_rows_per_dst=[4298, 4370, 4942, 5458, 6202, 6522, 7034, 5826] dup_rows_per_dst=[806, 736, 1156, 2356, 4446, 3698, 4736, 2950] combine_rows_per_src=[4298, 4370, 4942, 5458, 6202, 6522, 7034, 5826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43732 payload_rows_remote=37012 dup_rows=21804 dup_rows_remote=16694 dedup_ratio=0.332703 router_tokens_per_expert=[1271, 1006, 700, 1032, 162, 1216, 449, 79, 175, 683, 1574, 291, 4, 1144, 71, 752, 1147, 342, 11, 29, 865, 450, 690, 490, 328, 1445, 295, 772, 424, 122, 324, 7, 861, 430, 1219, 7, 1202, 0, 1256, 114, 409, 850, 728, 530, 0, 749, 52, 359, 134, 192, 424, 186, 207, 1104, 115, 864, 442, 685, 370, 231, 357, 140, 198, 3] payload_rows_per_dst=[6720, 6106, 6342, 5276, 5316, 5194, 4892, 3886] dup_rows_per_dst=[5110, 3282, 4218, 2158, 2350, 2160, 1560, 966] combine_rows_per_src=[6720, 6106, 6342, 5276, 5316, 5194, 4892, 3886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43592 payload_rows_remote=37566 dup_rows=21944 dup_rows_remote=18456 dedup_ratio=0.334839 router_tokens_per_expert=[1062, 1125, 463, 560, 117, 1069, 446, 161, 401, 1223, 851, 686, 2, 1243, 4, 1058, 646, 915, 3, 27, 584, 425, 548, 374, 355, 529, 655, 694, 198, 424, 1138, 1, 1059, 461, 484, 1, 526, 7, 909, 155, 564, 459, 263, 754, 0, 598, 149, 628, 592, 189, 97, 973, 170, 744, 30, 900, 421, 869, 387, 530, 297, 1326, 239, 0] payload_rows_per_dst=[3052, 5458, 5328, 6436, 5116, 6064, 6026, 6112] dup_rows_per_dst=[460, 2358, 1716, 3690, 2508, 3252, 3488, 4472] combine_rows_per_src=[3052, 5458, 5328, 6436, 5116, 6064, 6026, 6112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38720 dup_rows=21142 dup_rows_remote=18828 dedup_ratio=0.322601 router_tokens_per_expert=[1203, 955, 463, 925, 119, 1133, 418, 151, 264, 1838, 1291, 321, 6, 927, 32, 1236, 622, 715, 7, 44, 691, 477, 273, 399, 240, 810, 463, 1154, 164, 86, 452, 1, 1130, 743, 603, 18, 738, 0, 1241, 180, 515, 312, 437, 651, 20, 672, 172, 721, 342, 151, 331, 547, 173, 777, 68, 996, 380, 847, 545, 415, 414, 443, 304, 2] payload_rows_per_dst=[6204, 6950, 5674, 5206, 5342, 4872, 5320, 4826] dup_rows_per_dst=[3678, 4880, 2314, 1652, 2432, 2128, 2184, 1874] combine_rows_per_src=[6204, 6950, 5674, 5206, 5342, 4872, 5320, 4826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281974784 dispatch_meta_bytes=231728 combine_payload_bytes=281974784 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=282206512 ingress_bytes=502175744 link_bytes=502175744 token_bytes=564181296 weight_movement_bytes=220200960 total_nvlink_bytes=784382256 weight_to_token_ratio=0.390302 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.733662125e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275681280 dispatch_meta_bytes=223296 combine_payload_bytes=275681280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=275904576 ingress_bytes=385781760 link_bytes=385781760 token_bytes=551585856 weight_movement_bytes=110100480 total_nvlink_bytes=661686336 weight_to_token_ratio=0.199607 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.709379308e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=227040 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278531808 ingress_bytes=388405248 link_bytes=388405248 token_bytes=556836576 weight_movement_bytes=110100480 total_nvlink_bytes=666937056 weight_to_token_ratio=0.197725 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755006080e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=265302016 dispatch_meta_bytes=214824 combine_payload_bytes=265302016 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=265516840 ingress_bytes=705703936 link_bytes=705703936 token_bytes=530818856 weight_movement_bytes=440401920 total_nvlink_bytes=971220776 weight_to_token_ratio=0.829665 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=6912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.227335213e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=269273088 dispatch_meta_bytes=224088 combine_payload_bytes=269273088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=269497176 ingress_bytes=379373568 link_bytes=379373568 token_bytes=538770264 weight_movement_bytes=110100480 total_nvlink_bytes=648870744 weight_to_token_ratio=0.204355 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.597930310e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44302 payload_rows_remote=38814 dup_rows=21234 dup_rows_remote=18962 dedup_ratio=0.324005 router_tokens_per_expert=[1105, 1339, 687, 778, 100, 867, 400, 75, 529, 1205, 997, 468, 0, 1078, 29, 1195, 570, 1014, 25, 38, 1095, 307, 429, 708, 474, 471, 346, 619, 297, 386, 411, 9, 627, 665, 751, 42, 906, 0, 1063, 220, 607, 671, 302, 960, 16, 480, 32, 320, 318, 198, 229, 436, 339, 938, 117, 760, 400, 1109, 596, 333, 284, 748, 249, 1] payload_rows_per_dst=[5004, 6562, 5892, 5488, 5446, 4908, 5940, 5062] dup_rows_per_dst=[1754, 4344, 2480, 2272, 3102, 1868, 2940, 2474] combine_rows_per_src=[5004, 6562, 5892, 5488, 5446, 4908, 5940, 5062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277544960 dispatch_meta_bytes=230192 combine_payload_bytes=277544960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277775152 ingress_bytes=387645440 link_bytes=387645440 token_bytes=555320112 weight_movement_bytes=110100480 total_nvlink_bytes=665420592 weight_to_token_ratio=0.198265 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.741791769e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52932 payload_rows_remote=45526 dup_rows=12604 dup_rows_remote=12604 dedup_ratio=0.192322 router_tokens_per_expert=[0, 0, 0, 344, 3703, 0, 437, 1254, 2059, 614, 42, 0, 0, 49, 0, 859, 0, 297, 0, 0, 2760, 3234, 0, 0, 0, 0, 2377, 0, 45, 555, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3426, 0, 0, 0, 157, 0, 0, 1796, 2582, 0, 480, 0, 0, 0, 2922, 0, 0, 7, 1, 6, 19, 2742] payload_rows_per_dst=[7104, 5958, 5748, 5382, 7406, 6916, 6772, 7646] dup_rows_per_dst=[3434, 1288, 366, 574, 0, 250, 2944, 3748] combine_rows_per_src=[7104, 5958, 5748, 5382, 7406, 6916, 6772, 7646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51956 payload_rows_remote=45004 dup_rows=13580 dup_rows_remote=13454 dedup_ratio=0.207214 router_tokens_per_expert=[0, 0, 0, 308, 3835, 0, 512, 1568, 1196, 982, 26, 0, 0, 110, 0, 1074, 2, 222, 0, 0, 2628, 3163, 0, 1, 0, 0, 2365, 2, 83, 1241, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 4, 3405, 0, 0, 0, 130, 0, 0, 1084, 2658, 2, 606, 1, 1, 2, 2765, 1, 3, 2, 1, 1, 4, 2775] payload_rows_per_dst=[4920, 5282, 6914, 6068, 7712, 6952, 6558, 7550] dup_rows_per_dst=[1618, 1494, 2962, 1314, 362, 126, 2150, 3554] combine_rows_per_src=[4920, 5282, 6914, 6068, 7712, 6952, 6558, 7550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50394 payload_rows_remote=42870 dup_rows=15142 dup_rows_remote=11764 dedup_ratio=0.231049 router_tokens_per_expert=[0, 0, 1, 328, 3234, 1, 295, 1675, 1994, 695, 155, 0, 0, 54, 0, 865, 0, 116, 0, 1, 3052, 2830, 1, 0, 0, 0, 2973, 1, 131, 819, 0, 0, 0, 0, 0, 0, 1, 4, 0, 1, 0, 0, 3738, 0, 0, 0, 65, 0, 0, 1351, 2504, 0, 429, 1, 0, 2, 2623, 0, 1, 7, 18, 10, 50, 2742] payload_rows_per_dst=[1234, 5922, 7796, 6646, 7366, 7494, 6412, 7524] dup_rows_per_dst=[16, 1604, 4204, 1202, 2464, 112, 2162, 3378] combine_rows_per_src=[1234, 5922, 7796, 6646, 7366, 7494, 6412, 7524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47228 payload_rows_remote=39918 dup_rows=18308 dup_rows_remote=13424 dedup_ratio=0.279358 router_tokens_per_expert=[0, 0, 0, 485, 3550, 0, 237, 2247, 1383, 460, 10, 0, 0, 103, 0, 713, 0, 37, 0, 1, 3123, 3128, 0, 0, 0, 0, 2966, 0, 62, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2890, 0, 0, 0, 55, 0, 0, 1238, 2890, 0, 576, 0, 0, 0, 3183, 0, 0, 0, 6, 0, 8, 2310] payload_rows_per_dst=[7310, 7706, 6678, 7488, 7100, 7714, 3204, 28] dup_rows_per_dst=[4884, 3998, 1552, 3494, 0, 3956, 424, 0] combine_rows_per_src=[7310, 7706, 6678, 7488, 7100, 7714, 3204, 28] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49148 payload_rows_remote=42488 dup_rows=16388 dup_rows_remote=13562 dedup_ratio=0.250061 router_tokens_per_expert=[0, 0, 0, 177, 3927, 0, 738, 1515, 1431, 724, 64, 1, 0, 117, 0, 320, 2, 13, 1, 0, 2790, 3296, 0, 0, 0, 0, 2298, 0, 50, 469, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3503, 0, 0, 1, 7, 0, 1, 1741, 2445, 0, 556, 0, 0, 0, 2860, 0, 0, 1, 6, 1, 11, 3701] payload_rows_per_dst=[1822, 4606, 7876, 5148, 8000, 7012, 6660, 8024] dup_rows_per_dst=[8, 708, 4328, 486, 2886, 10, 2826, 5136] combine_rows_per_src=[1822, 4606, 7876, 5148, 8000, 7012, 6660, 8024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278218752 dispatch_meta_bytes=231104 combine_payload_bytes=278218752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278449856 ingress_bytes=388319232 link_bytes=388319232 token_bytes=556668608 weight_movement_bytes=110100480 total_nvlink_bytes=666769088 weight_to_token_ratio=0.197785 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753510120e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=326330368 dispatch_meta_bytes=232520 combine_payload_bytes=326330368 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=326562888 ingress_bytes=546531328 link_bytes=546531328 token_bytes=652893256 weight_movement_bytes=220200960 total_nvlink_bytes=873094216 weight_to_token_ratio=0.337269 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.505078683e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=322588672 dispatch_meta_bytes=233832 combine_payload_bytes=322588672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=322822504 ingress_bytes=432689152 link_bytes=432689152 token_bytes=645411176 weight_movement_bytes=110100480 total_nvlink_bytes=755511656 weight_to_token_ratio=0.170590 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=27648 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.525176005e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=307292160 dispatch_meta_bytes=218536 combine_payload_bytes=307292160 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=307510696 ingress_bytes=747694080 link_bytes=747694080 token_bytes=614802856 weight_movement_bytes=440401920 total_nvlink_bytes=1055204776 weight_to_token_ratio=0.716330 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.300362980e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52382 payload_rows_remote=46912 dup_rows=13154 dup_rows_remote=13028 dedup_ratio=0.200714 router_tokens_per_expert=[0, 0, 0, 286, 3837, 0, 490, 1843, 1373, 1064, 54, 0, 1, 264, 0, 689, 4, 88, 0, 0, 2706, 3108, 0, 0, 0, 0, 2379, 1, 51, 734, 0, 2, 0, 0, 1, 0, 0, 1, 0, 3, 0, 0, 3362, 1, 0, 0, 102, 0, 0, 1385, 2517, 0, 529, 2, 0, 0, 2929, 1, 0, 4, 13, 1, 16, 2927] payload_rows_per_dst=[7114, 5464, 5470, 5996, 7682, 6822, 7030, 6804] dup_rows_per_dst=[4340, 1426, 126, 1528, 2, 108, 3792, 1832] combine_rows_per_src=[7114, 5464, 5470, 5996, 7682, 6822, 7030, 6804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=286132224 dispatch_meta_bytes=213368 combine_payload_bytes=286132224 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=286345592 ingress_bytes=616433664 link_bytes=616433664 token_bytes=572477816 weight_movement_bytes=330301440 total_nvlink_bytes=902779256 weight_to_token_ratio=0.576968 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.072079528e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=304553984 dispatch_meta_bytes=224200 combine_payload_bytes=304553984 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=304778184 ingress_bytes=524754944 link_bytes=524754944 token_bytes=609332168 weight_movement_bytes=220200960 total_nvlink_bytes=829533128 weight_to_token_ratio=0.361381 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.126351550e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43708 payload_rows_remote=38132 dup_rows=21828 dup_rows_remote=18878 dedup_ratio=0.333069 router_tokens_per_expert=[1274, 1446, 776, 699, 455, 1456, 379, 89, 470, 1364, 308, 378, 29, 874, 112, 728, 683, 505, 23, 49, 858, 290, 352, 1087, 434, 612, 547, 860, 602, 151, 566, 6, 736, 606, 1060, 6, 590, 3, 1183, 203, 327, 724, 616, 796, 16, 610, 117, 501, 115, 554, 322, 305, 323, 519, 86, 571, 558, 592, 645, 402, 278, 666, 271, 5] payload_rows_per_dst=[7014, 5576, 6340, 5384, 4734, 5282, 4348, 5030] dup_rows_per_dst=[6134, 2950, 3720, 2172, 1674, 2132, 1242, 1804] combine_rows_per_src=[7014, 5576, 6340, 5384, 4734, 5282, 4348, 5030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=336265216 dispatch_meta_bytes=239760 combine_payload_bytes=336265216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=336504976 ingress_bytes=556466176 link_bytes=556466176 token_bytes=672770192 weight_movement_bytes=220200960 total_nvlink_bytes=892971152 weight_to_token_ratio=0.327305 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.677862030e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56648 payload_rows_remote=49428 dup_rows=8888 dup_rows_remote=6106 dedup_ratio=0.135620 router_tokens_per_expert=[0, 0, 0, 617, 2, 0, 4093, 0, 0, 706, 0, 0, 0, 0, 0, 0, 2726, 0, 0, 0, 0, 4082, 1715, 0, 0, 0, 2784, 4096, 0, 2078, 0, 0, 0, 1, 0, 0, 0, 2104, 170, 0, 3972, 0, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 2888, 0, 0, 0, 0, 1] payload_rows_per_dst=[8190, 8180, 3430, 7210, 7220, 7984, 8192, 6242] dup_rows_per_dst=[1234, 1396, 0, 2514, 2782, 578, 0, 384] combine_rows_per_src=[8190, 8180, 3430, 7210, 7220, 7984, 8192, 6242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48502 payload_rows_remote=40312 dup_rows=17034 dup_rows_remote=9504 dedup_ratio=0.259918 router_tokens_per_expert=[0, 0, 0, 583, 0, 0, 4090, 0, 0, 828, 0, 0, 0, 0, 0, 0, 2529, 0, 0, 0, 0, 4073, 1502, 0, 0, 0, 2642, 4096, 0, 2244, 0, 0, 0, 2, 0, 0, 0, 2151, 184, 2, 3790, 0, 281, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 625, 1, 3141, 1, 0, 0, 0, 2] payload_rows_per_dst=[1166, 1656, 8170, 6268, 7220, 7640, 8192, 8190] dup_rows_per_dst=[0, 0, 2980, 1800, 2516, 2206, 2, 7530] combine_rows_per_src=[1166, 1656, 8170, 6268, 7220, 7640, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57488 payload_rows_remote=49510 dup_rows=8048 dup_rows_remote=7694 dedup_ratio=0.122803 router_tokens_per_expert=[0, 1, 0, 699, 1, 0, 4078, 0, 0, 646, 0, 0, 0, 0, 0, 0, 2381, 0, 0, 0, 0, 4079, 2299, 1, 0, 0, 3214, 4094, 0, 1198, 0, 0, 0, 4, 0, 0, 0, 2370, 301, 0, 3975, 2, 187, 0, 0, 1, 0, 1, 0, 0, 3, 1, 0, 0, 0, 0, 601, 1, 2624, 0, 0, 0, 1, 5] payload_rows_per_dst=[8164, 8166, 4598, 7082, 7364, 7978, 8192, 5944] dup_rows_per_dst=[1394, 1284, 2, 1742, 2748, 354, 4, 520] combine_rows_per_src=[8164, 8166, 4598, 7082, 7364, 7978, 8192, 5944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52970 payload_rows_remote=46588 dup_rows=12566 dup_rows_remote=11332 dedup_ratio=0.191742 router_tokens_per_expert=[0, 0, 0, 332, 3590, 0, 622, 1566, 1735, 822, 208, 0, 0, 240, 0, 901, 1, 114, 0, 1, 2572, 3006, 0, 0, 0, 0, 2579, 0, 74, 1154, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3748, 1, 0, 0, 88, 0, 0, 1263, 2751, 0, 666, 0, 0, 0, 2136, 0, 0, 5, 5, 11, 36, 2539] payload_rows_per_dst=[7050, 5890, 5230, 6382, 7182, 7546, 6856, 6834] dup_rows_per_dst=[4002, 1922, 146, 1234, 0, 128, 2504, 2630] combine_rows_per_src=[7050, 5890, 5230, 6382, 7182, 7546, 6856, 6834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50696 payload_rows_remote=42506 dup_rows=14840 dup_rows_remote=9034 dedup_ratio=0.226440 router_tokens_per_expert=[0, 0, 0, 1162, 0, 0, 4091, 0, 0, 1466, 0, 0, 0, 0, 0, 0, 2218, 0, 0, 0, 0, 4094, 1573, 0, 0, 0, 2968, 4096, 0, 2149, 0, 0, 0, 0, 0, 0, 0, 1492, 123, 0, 4063, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 2779, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8190, 7894, 4298, 3152, 4810, 8192, 5970] dup_rows_per_dst=[5806, 2930, 5624, 0, 78, 148, 0, 254] combine_rows_per_src=[8190, 8190, 7894, 4298, 3152, 4810, 8192, 5970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51700 payload_rows_remote=43508 dup_rows=13836 dup_rows_remote=13836 dedup_ratio=0.211121 router_tokens_per_expert=[0, 0, 0, 978, 0, 0, 4088, 0, 0, 1208, 0, 0, 1, 0, 0, 0, 3150, 0, 0, 0, 0, 4087, 1563, 0, 0, 0, 3037, 4095, 0, 1556, 3, 0, 0, 1, 0, 0, 0, 1745, 360, 1, 4056, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 225, 0, 2542, 0, 0, 1, 0, 0] payload_rows_per_dst=[1990, 3038, 7598, 7146, 7442, 8112, 8192, 8182] dup_rows_per_dst=[144, 366, 2716, 2046, 3072, 140, 0, 5352] combine_rows_per_src=[1990, 3038, 7598, 7146, 7442, 8112, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=354299904 dispatch_meta_bytes=222136 combine_payload_bytes=354299904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=354522040 ingress_bytes=464400384 link_bytes=464400384 token_bytes=708821944 weight_movement_bytes=110100480 total_nvlink_bytes=818922424 weight_to_token_ratio=0.155329 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=15616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.076686485e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=273330176 dispatch_meta_bytes=228040 combine_payload_bytes=273330176 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=273558216 ingress_bytes=603631616 link_bytes=603631616 token_bytes=546888392 weight_movement_bytes=330301440 total_nvlink_bytes=877189832 weight_to_token_ratio=0.603965 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.049814661e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=333942784 dispatch_meta_bytes=231680 combine_payload_bytes=333942784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=334174464 ingress_bytes=444043264 link_bytes=444043264 token_bytes=668117248 weight_movement_bytes=110100480 total_nvlink_bytes=778217728 weight_to_token_ratio=0.164792 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.722642687e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=288956416 dispatch_meta_bytes=199264 combine_payload_bytes=288956416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=289155680 ingress_bytes=399056896 link_bytes=399056896 token_bytes=578112096 weight_movement_bytes=110100480 total_nvlink_bytes=688212576 weight_to_token_ratio=0.190448 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.940255757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56298 payload_rows_remote=50774 dup_rows=9238 dup_rows_remote=8094 dedup_ratio=0.140961 router_tokens_per_expert=[0, 0, 0, 696, 5, 0, 4064, 0, 0, 682, 0, 0, 0, 0, 0, 0, 2802, 0, 0, 0, 0, 4094, 2296, 0, 0, 0, 3471, 4094, 1, 1913, 1, 0, 0, 5, 0, 2, 1, 1784, 172, 5, 3970, 0, 172, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 2165, 1, 0, 0, 0, 0] payload_rows_per_dst=[8140, 8190, 5524, 7678, 5736, 7948, 8188, 4894] dup_rows_per_dst=[1390, 1362, 1144, 3094, 1730, 338, 0, 180] combine_rows_per_src=[8140, 8190, 5524, 7678, 5736, 7948, 8188, 4894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=354887680 dispatch_meta_bytes=228816 combine_payload_bytes=354887680 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=355116496 ingress_bytes=575088640 link_bytes=575088640 token_bytes=710004176 weight_movement_bytes=220200960 total_nvlink_bytes=930205136 weight_to_token_ratio=0.310140 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.000173731e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=304683008 dispatch_meta_bytes=206160 combine_payload_bytes=304683008 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=304889168 ingress_bytes=524883968 link_bytes=524883968 token_bytes=609572176 weight_movement_bytes=220200960 total_nvlink_bytes=829773136 weight_to_token_ratio=0.361239 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=24448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.128595489e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=311865344 dispatch_meta_bytes=229376 combine_payload_bytes=311865344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=312094720 ingress_bytes=421965824 link_bytes=421965824 token_bytes=623960064 weight_movement_bytes=110100480 total_nvlink_bytes=734060544 weight_to_token_ratio=0.176454 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.338679694e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=363948032 dispatch_meta_bytes=235472 combine_payload_bytes=363948032 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=364183504 ingress_bytes=694249472 link_bytes=694249472 token_bytes=728131536 weight_movement_bytes=330301440 total_nvlink_bytes=1058432976 weight_to_token_ratio=0.453629 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=14336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.207414016e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62038 payload_rows_remote=54688 dup_rows=3498 dup_rows_remote=2038 dedup_ratio=0.053375 router_tokens_per_expert=[0, 0, 897, 0, 0, 3948, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 3546, 0, 0, 0, 4036, 0, 0, 4015, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 2298, 901, 0, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 3723, 4096, 0] payload_rows_per_dst=[8104, 8082, 7092, 7736, 7350, 8030, 8192, 7452] dup_rows_per_dst=[1586, 304, 0, 0, 1460, 0, 58, 90] combine_rows_per_src=[8104, 8082, 7092, 7736, 7350, 8030, 8192, 7452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53924 payload_rows_remote=46420 dup_rows=11612 dup_rows_remote=6586 dedup_ratio=0.177185 router_tokens_per_expert=[0, 0, 1082, 0, 0, 3879, 0, 0, 0, 0, 0, 177, 2, 0, 0, 0, 0, 0, 3319, 0, 0, 0, 3918, 0, 0, 4014, 0, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 2634, 1269, 0, 877, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 3535, 4094, 0] payload_rows_per_dst=[2164, 7474, 7338, 6244, 7518, 8046, 7636, 7504] dup_rows_per_dst=[0, 280, 2722, 0, 2042, 1482, 60, 5026] combine_rows_per_src=[2164, 7474, 7338, 6244, 7518, 8046, 7636, 7504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=392003584 dispatch_meta_bytes=226904 combine_payload_bytes=392003584 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=392230488 ingress_bytes=612204544 link_bytes=612204544 token_bytes=784234072 weight_movement_bytes=220200960 total_nvlink_bytes=1004435032 weight_to_token_ratio=0.280785 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=16000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.064724392e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59536 payload_rows_remote=51500 dup_rows=6000 dup_rows_remote=5998 dedup_ratio=0.091553 router_tokens_per_expert=[0, 0, 1578, 0, 0, 3958, 0, 0, 1, 0, 0, 181, 1, 0, 1, 0, 0, 0, 2948, 0, 0, 0, 3966, 0, 0, 4017, 0, 3769, 0, 1, 5, 0, 0, 0, 0, 0, 2248, 1019, 0, 1011, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 30, 0, 1, 0, 0, 0, 43, 0, 0, 0, 0, 3897, 4091, 0] payload_rows_per_dst=[6194, 7956, 6806, 7540, 7014, 8036, 8190, 7800] dup_rows_per_dst=[1746, 344, 2222, 10, 1542, 2, 54, 80] combine_rows_per_src=[6194, 7956, 6806, 7540, 7014, 8036, 8190, 7800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55644 payload_rows_remote=48928 dup_rows=9892 dup_rows_remote=8052 dedup_ratio=0.150940 router_tokens_per_expert=[0, 0, 0, 817, 0, 0, 3989, 0, 0, 1085, 0, 0, 0, 0, 0, 0, 2878, 0, 0, 1, 0, 4095, 1658, 0, 0, 0, 2820, 4096, 0, 1458, 0, 0, 0, 3, 0, 0, 0, 1901, 303, 2, 3975, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 851, 0, 2523, 0, 0, 0, 0, 0] payload_rows_per_dst=[8030, 8192, 3318, 6716, 7298, 7960, 8192, 5938] dup_rows_per_dst=[1582, 2168, 0, 1840, 2876, 616, 0, 810] combine_rows_per_src=[8030, 8192, 3318, 6716, 7298, 7960, 8192, 5938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58130 payload_rows_remote=49996 dup_rows=7406 dup_rows_remote=3148 dedup_ratio=0.113007 router_tokens_per_expert=[0, 0, 1563, 0, 0, 3872, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 3462, 0, 0, 0, 3934, 0, 0, 3978, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 2186, 786, 0, 999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 3788, 4096, 0] payload_rows_per_dst=[8134, 7872, 6924, 8030, 3446, 7956, 8192, 7576] dup_rows_per_dst=[4258, 146, 0, 2718, 124, 0, 86, 74] combine_rows_per_src=[8134, 7872, 6924, 8030, 3446, 7956, 8192, 7576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56920 payload_rows_remote=48730 dup_rows=8616 dup_rows_remote=8540 dedup_ratio=0.131470 router_tokens_per_expert=[0, 0, 712, 0, 0, 3975, 0, 0, 0, 0, 0, 94, 1, 0, 0, 0, 0, 0, 3490, 0, 0, 0, 4041, 0, 0, 3990, 0, 3904, 0, 0, 1, 0, 0, 0, 0, 0, 1781, 1313, 0, 1418, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 3850, 4095, 0] payload_rows_per_dst=[1424, 8104, 8146, 7810, 7530, 7982, 8190, 7734] dup_rows_per_dst=[0, 168, 6784, 0, 1494, 6, 76, 88] combine_rows_per_src=[1424, 8104, 8146, 7810, 7530, 7982, 8190, 7734] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47902 payload_rows_remote=40534 dup_rows=17634 dup_rows_remote=12734 dedup_ratio=0.269073 router_tokens_per_expert=[1, 0, 0, 381, 3677, 0, 536, 1555, 1938, 1045, 279, 0, 0, 148, 0, 1372, 6, 150, 0, 3, 2911, 2867, 0, 0, 6, 0, 1962, 4, 59, 1421, 0, 8, 0, 0, 0, 0, 5, 2, 0, 3, 0, 0, 3517, 1, 0, 2, 149, 1, 0, 1122, 2307, 0, 717, 0, 0, 0, 1589, 10, 0, 58, 44, 58, 17, 2837] payload_rows_per_dst=[6978, 7368, 5928, 7650, 7370, 7566, 4218, 824] dup_rows_per_dst=[3702, 4900, 212, 4944, 4, 2824, 1024, 24] combine_rows_per_src=[6978, 7368, 5928, 7650, 7370, 7566, 4218, 824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62422 payload_rows_remote=55784 dup_rows=3114 dup_rows_remote=3114 dedup_ratio=0.047516 router_tokens_per_expert=[0, 0, 834, 0, 0, 3997, 0, 0, 2, 0, 0, 27, 5, 0, 2, 0, 0, 0, 3319, 0, 0, 0, 4080, 0, 0, 4039, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 2593, 748, 0, 1026, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 31, 2, 0, 0, 0, 0, 80, 0, 0, 0, 0, 3972, 4091, 0] payload_rows_per_dst=[8116, 8166, 6638, 7826, 7432, 8078, 8186, 7980] dup_rows_per_dst=[1546, 66, 0, 0, 1302, 14, 62, 124] combine_rows_per_src=[8116, 8166, 6638, 7826, 7432, 8078, 8186, 7980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=332738560 dispatch_meta_bytes=212024 combine_payload_bytes=332738560 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=332950584 ingress_bytes=552939520 link_bytes=552939520 token_bytes=665689144 weight_movement_bytes=220200960 total_nvlink_bytes=885890104 weight_to_token_ratio=0.330786 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.616527682e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=369152000 dispatch_meta_bytes=229992 combine_payload_bytes=369152000 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=369381992 ingress_bytes=589352960 link_bytes=589352960 token_bytes=738533992 weight_movement_bytes=220200960 total_nvlink_bytes=958734952 weight_to_token_ratio=0.298160 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.024981729e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=350715904 dispatch_meta_bytes=227920 combine_payload_bytes=350715904 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=350943824 ingress_bytes=681017344 link_bytes=681017344 token_bytes=701659728 weight_movement_bytes=330301440 total_nvlink_bytes=1031961168 weight_to_token_ratio=0.470743 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.184401169e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=358371328 dispatch_meta_bytes=212576 combine_payload_bytes=358371328 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=358583904 ingress_bytes=688672768 link_bytes=688672768 token_bytes=716955232 weight_movement_bytes=330301440 total_nvlink_bytes=1047256672 weight_to_token_ratio=0.460700 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=21888 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.197715211e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=349296640 dispatch_meta_bytes=229080 combine_payload_bytes=349296640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349525720 ingress_bytes=459397120 link_bytes=459397120 token_bytes=698822360 weight_movement_bytes=110100480 total_nvlink_bytes=808922840 weight_to_token_ratio=0.157551 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.989671495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=290547712 dispatch_meta_bytes=213072 combine_payload_bytes=290547712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=290760784 ingress_bytes=400648192 link_bytes=400648192 token_bytes=581308496 weight_movement_bytes=110100480 total_nvlink_bytes=691408976 weight_to_token_ratio=0.189401 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=13184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.967931012e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=399859712 dispatch_meta_bytes=235592 combine_payload_bytes=399859712 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=400095304 ingress_bytes=620060672 link_bytes=620060672 token_bytes=799955016 weight_movement_bytes=220200960 total_nvlink_bytes=1020155976 weight_to_token_ratio=0.275267 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=25600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.078387491e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56262 payload_rows_remote=48080 dup_rows=9274 dup_rows_remote=9274 dedup_ratio=0.141510 router_tokens_per_expert=[0, 3, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3431, 0, 0, 0, 0, 0, 0, 2686, 199, 0, 0, 0, 0, 4094, 4091, 0, 4095, 0, 0, 0, 0, 1, 369, 2766, 0, 0, 3798, 0, 0, 0, 4, 0, 0, 0, 1186, 3700, 0, 0, 0, 321, 0, 0, 0, 1838, 0] payload_rows_per_dst=[8192, 8190, 6862, 7106, 8182, 6838, 5060, 5832] dup_rows_per_dst=[374, 0, 0, 3792, 0, 1902, 986, 2220] combine_rows_per_src=[8192, 8190, 6862, 7106, 8182, 6838, 5060, 5832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=344637440 dispatch_meta_bytes=229416 combine_payload_bytes=344637440 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=344866856 ingress_bytes=785039360 link_bytes=785039360 token_bytes=689504296 weight_movement_bytes=440401920 total_nvlink_bytes=1129906216 weight_to_token_ratio=0.638723 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.365312564e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54110 payload_rows_remote=46126 dup_rows=11426 dup_rows_remote=5922 dedup_ratio=0.174347 router_tokens_per_expert=[0, 3, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3150, 0, 0, 0, 0, 0, 0, 1046, 123, 0, 0, 0, 0, 4091, 4042, 0, 4094, 0, 0, 0, 0, 2, 262, 3032, 0, 0, 3918, 0, 0, 0, 20, 0, 0, 0, 2124, 3522, 0, 0, 0, 317, 0, 0, 0, 2905, 0] payload_rows_per_dst=[6118, 8188, 6300, 2322, 8118, 8036, 7044, 7984] dup_rows_per_dst=[164, 0, 0, 16, 560, 2856, 2326, 5504] combine_rows_per_src=[6118, 8188, 6300, 2322, 8118, 8036, 7044, 7984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54744 payload_rows_remote=46582 dup_rows=10792 dup_rows_remote=4506 dedup_ratio=0.164673 router_tokens_per_expert=[0, 10, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3647, 0, 0, 0, 0, 0, 0, 1329, 112, 0, 0, 0, 0, 4087, 4091, 0, 4091, 0, 0, 0, 0, 5, 310, 2919, 0, 0, 3989, 0, 1, 0, 9, 0, 0, 0, 1937, 3750, 0, 0, 0, 263, 0, 0, 0, 2068, 0] payload_rows_per_dst=[8192, 8182, 7294, 2882, 8182, 8162, 3886, 7964] dup_rows_per_dst=[302, 0, 0, 0, 0, 6286, 6, 4198] combine_rows_per_src=[8192, 8182, 7294, 2882, 8182, 8162, 3886, 7964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59930 payload_rows_remote=52122 dup_rows=5606 dup_rows_remote=5604 dedup_ratio=0.085541 router_tokens_per_expert=[0, 0, 1734, 0, 0, 3807, 0, 0, 2, 0, 0, 297, 0, 0, 0, 0, 0, 0, 2773, 0, 0, 0, 3941, 0, 0, 3979, 0, 3904, 0, 0, 1, 0, 0, 0, 0, 0, 2765, 1170, 0, 640, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 3547, 4095, 0] payload_rows_per_dst=[8068, 7946, 5546, 7808, 7284, 7958, 8192, 7128] dup_rows_per_dst=[3014, 534, 0, 2, 1866, 6, 72, 112] combine_rows_per_src=[8068, 7946, 5546, 7808, 7284, 7958, 8192, 7128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53542 payload_rows_remote=45350 dup_rows=11994 dup_rows_remote=11564 dedup_ratio=0.183014 router_tokens_per_expert=[0, 0, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3383, 0, 0, 0, 0, 0, 0, 2006, 84, 0, 0, 0, 0, 4096, 4082, 0, 4096, 0, 0, 0, 0, 0, 226, 2636, 0, 0, 3987, 0, 0, 0, 5, 0, 0, 0, 1869, 3596, 0, 0, 0, 222, 0, 0, 0, 2265, 0] payload_rows_per_dst=[8192, 8192, 7896, 8150, 8066, 452, 7846, 4748] dup_rows_per_dst=[430, 98, 4142, 4004, 0, 0, 3094, 226] combine_rows_per_src=[8192, 8192, 7896, 8150, 8066, 452, 7846, 4748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55250 payload_rows_remote=51528 dup_rows=10286 dup_rows_remote=10282 dedup_ratio=0.156952 router_tokens_per_expert=[0, 4, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3768, 0, 0, 0, 0, 0, 0, 1671, 48, 0, 0, 0, 0, 4093, 4073, 0, 4095, 0, 0, 0, 0, 1, 392, 2763, 0, 0, 4007, 0, 0, 0, 5, 0, 0, 0, 1858, 3703, 0, 0, 0, 156, 0, 0, 0, 2079, 0] payload_rows_per_dst=[8192, 8190, 7536, 3430, 8146, 8180, 3722, 7854] dup_rows_per_dst=[106, 0, 0, 8, 0, 6146, 4, 4022] combine_rows_per_src=[8192, 8190, 7536, 3430, 8146, 8180, 3722, 7854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59148 payload_rows_remote=51528 dup_rows=6388 dup_rows_remote=6388 dedup_ratio=0.097473 router_tokens_per_expert=[0, 9, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3809, 0, 0, 0, 0, 0, 0, 1418, 105, 0, 0, 0, 0, 4079, 4086, 0, 4090, 0, 0, 0, 0, 0, 431, 2874, 0, 0, 4008, 0, 1, 0, 0, 0, 0, 0, 1555, 3661, 0, 0, 0, 281, 0, 0, 0, 2218, 0] payload_rows_per_dst=[8176, 8180, 7620, 8158, 8172, 6206, 7880, 4756] dup_rows_per_dst=[284, 0, 0, 2904, 0, 406, 2552, 242] combine_rows_per_src=[8176, 8180, 7620, 8158, 8172, 6206, 7880, 4756] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53766 payload_rows_remote=45624 dup_rows=11770 dup_rows_remote=9494 dedup_ratio=0.179596 router_tokens_per_expert=[0, 0, 0, 1008, 18, 0, 4032, 0, 0, 1139, 0, 0, 0, 0, 0, 0, 2236, 0, 0, 0, 0, 4070, 1665, 1, 0, 0, 3441, 4095, 0, 1976, 1, 0, 1, 24, 0, 0, 0, 1961, 142, 23, 3904, 3, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 2014, 0, 0, 0, 0, 0] payload_rows_per_dst=[8108, 8142, 7398, 5044, 4130, 7892, 8190, 4862] dup_rows_per_dst=[2008, 2276, 5140, 1058, 172, 832, 0, 284] combine_rows_per_src=[8108, 8142, 7398, 5044, 4130, 7892, 8190, 4862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=330631168 dispatch_meta_bytes=208192 combine_payload_bytes=330631168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=330839360 ingress_bytes=440731648 link_bytes=440731648 token_bytes=661470528 weight_movement_bytes=110100480 total_nvlink_bytes=771571008 weight_to_token_ratio=0.166448 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.665048238e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=333899776 dispatch_meta_bytes=204352 combine_payload_bytes=333899776 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=334104128 ingress_bytes=774301696 link_bytes=774301696 token_bytes=668003904 weight_movement_bytes=440401920 total_nvlink_bytes=1108405824 weight_to_token_ratio=0.659280 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=16384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.346638000e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=325068800 dispatch_meta_bytes=227656 combine_payload_bytes=325068800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=325296456 ingress_bytes=435169280 link_bytes=435169280 token_bytes=650365256 weight_movement_bytes=110100480 total_nvlink_bytes=760465736 weight_to_token_ratio=0.169290 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.568309510e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=373610496 dispatch_meta_bytes=230904 combine_payload_bytes=373610496 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=373841400 ingress_bytes=703911936 link_bytes=703911936 token_bytes=747451896 weight_movement_bytes=330301440 total_nvlink_bytes=1077753336 weight_to_token_ratio=0.441903 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.224218630e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59258 payload_rows_remote=52276 dup_rows=6278 dup_rows_remote=6278 dedup_ratio=0.095795 router_tokens_per_expert=[0, 3954, 0, 3168, 0, 1167, 0, 852, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 13, 0, 0, 4086, 0, 0, 0, 0, 2150, 0, 0, 0, 0, 0, 0, 0, 0, 3491, 0, 0, 0, 0, 0, 4045, 0, 5, 84, 0, 0, 4096, 0, 0, 0, 131, 0, 0, 1417, 0, 0, 0, 0, 10, 0, 0, 0] payload_rows_per_dst=[7736, 8192, 8172, 5104, 6982, 8098, 7058, 7916] dup_rows_per_dst=[2638, 0, 32, 900, 0, 170, 2526, 12] combine_rows_per_src=[7736, 8192, 8172, 5104, 6982, 8098, 7058, 7916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=369352704 dispatch_meta_bytes=247240 combine_payload_bytes=369352704 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=369599944 ingress_bytes=589553664 link_bytes=589553664 token_bytes=738952648 weight_movement_bytes=220200960 total_nvlink_bytes=959153608 weight_to_token_ratio=0.297991 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.025330787e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=369352704 dispatch_meta_bytes=231664 combine_payload_bytes=369352704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369584368 ingress_bytes=479453184 link_bytes=479453184 token_bytes=738937072 weight_movement_bytes=110100480 total_nvlink_bytes=849037552 weight_to_token_ratio=0.148998 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.338479435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=327032832 dispatch_meta_bytes=220472 combine_payload_bytes=327032832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=327253304 ingress_bytes=437133312 link_bytes=437133312 token_bytes=654286136 weight_movement_bytes=110100480 total_nvlink_bytes=764386616 weight_to_token_ratio=0.168276 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=25088 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.602467256e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=374714368 dispatch_meta_bytes=234216 combine_payload_bytes=374714368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374948584 ingress_bytes=484814848 link_bytes=484814848 token_bytes=749662952 weight_movement_bytes=110100480 total_nvlink_bytes=859763432 weight_to_token_ratio=0.146867 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.431727590e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55320 payload_rows_remote=47230 dup_rows=10216 dup_rows_remote=8112 dedup_ratio=0.155884 router_tokens_per_expert=[0, 4034, 0, 3062, 0, 1103, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 3, 0, 0, 4066, 0, 0, 0, 0, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 3411, 0, 0, 0, 0, 0, 3991, 0, 5, 68, 0, 0, 4096, 0, 0, 0, 134, 0, 0, 1708, 0, 0, 0, 2, 1, 0, 0, 0] payload_rows_per_dst=[4440, 8188, 8132, 3118, 7160, 8000, 8192, 8090] dup_rows_per_dst=[624, 0, 10, 0, 1280, 128, 6070, 2104] combine_rows_per_src=[4440, 8188, 8132, 3118, 7160, 8000, 8192, 8090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58336 payload_rows_remote=50432 dup_rows=7200 dup_rows_remote=6876 dedup_ratio=0.109863 router_tokens_per_expert=[0, 3907, 0, 2838, 0, 1054, 0, 580, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 1, 4, 0, 0, 4059, 0, 0, 0, 0, 2119, 0, 2, 0, 0, 0, 0, 0, 0, 3609, 0, 0, 0, 0, 0, 3909, 0, 13, 192, 0, 0, 4094, 0, 0, 0, 114, 0, 0, 2152, 0, 0, 2, 4, 21, 0, 0, 0] payload_rows_per_dst=[6420, 8184, 8124, 4238, 7424, 7904, 8190, 7852] dup_rows_per_dst=[1374, 0, 8, 4, 944, 324, 4530, 16] combine_rows_per_src=[6420, 8184, 8124, 4238, 7424, 7904, 8190, 7852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60274 payload_rows_remote=52622 dup_rows=5262 dup_rows_remote=2820 dedup_ratio=0.080292 router_tokens_per_expert=[0, 3934, 0, 2856, 0, 919, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 3510, 0, 0, 0, 0, 0, 4032, 0, 2, 18, 0, 0, 4096, 0, 0, 0, 85, 0, 0, 2587, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[7652, 8192, 8162, 8192, 7020, 7880, 5306, 7870] dup_rows_per_dst=[2442, 34, 128, 2582, 0, 38, 38, 0] combine_rows_per_src=[7652, 8192, 8162, 8192, 7020, 7880, 5306, 7870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59034 payload_rows_remote=50880 dup_rows=6502 dup_rows_remote=3908 dedup_ratio=0.099213 router_tokens_per_expert=[0, 3, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3541, 0, 0, 0, 0, 0, 0, 1269, 137, 0, 0, 0, 0, 4084, 4005, 0, 4096, 0, 0, 0, 0, 0, 318, 3263, 0, 0, 3968, 0, 0, 0, 24, 0, 0, 0, 2173, 3065, 0, 0, 0, 383, 0, 0, 0, 2301, 0] payload_rows_per_dst=[8174, 8192, 7082, 8154, 8010, 6736, 7652, 5034] dup_rows_per_dst=[276, 0, 0, 2594, 0, 426, 2872, 334] combine_rows_per_src=[8174, 8192, 7082, 8154, 8010, 6736, 7652, 5034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56260 payload_rows_remote=48068 dup_rows=9276 dup_rows_remote=6098 dedup_ratio=0.141541 router_tokens_per_expert=[0, 3984, 0, 2998, 0, 959, 0, 948, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 4085, 0, 0, 0, 0, 2152, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 0, 0, 0, 0, 3986, 0, 4, 161, 0, 0, 4096, 0, 0, 0, 84, 0, 0, 1505, 0, 0, 0, 3, 6, 0, 0, 0] payload_rows_per_dst=[3422, 8192, 8170, 4304, 8008, 7992, 8192, 7980] dup_rows_per_dst=[392, 0, 8, 0, 5382, 310, 3178, 6] combine_rows_per_src=[3422, 8192, 8170, 4304, 8008, 7992, 8192, 7980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58418 payload_rows_remote=50266 dup_rows=7118 dup_rows_remote=7094 dedup_ratio=0.108612 router_tokens_per_expert=[0, 3944, 0, 2412, 0, 1021, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 8, 9, 0, 0, 4070, 0, 0, 0, 0, 2462, 0, 1, 0, 0, 0, 0, 0, 0, 3603, 0, 0, 0, 0, 0, 3960, 0, 21, 150, 0, 0, 4096, 0, 0, 0, 136, 0, 0, 1654, 0, 0, 0, 18, 14, 0, 0, 0] payload_rows_per_dst=[7194, 8192, 8152, 8192, 7206, 7972, 3576, 7934] dup_rows_per_dst=[1856, 0, 24, 4926, 0, 290, 4, 18] combine_rows_per_src=[7194, 8192, 8152, 8192, 7206, 7972, 3576, 7934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=338544640 dispatch_meta_bytes=221368 combine_payload_bytes=338544640 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=338766008 ingress_bytes=558745600 link_bytes=558745600 token_bytes=677310648 weight_movement_bytes=220200960 total_nvlink_bytes=897511608 weight_to_token_ratio=0.325111 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.717504963e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61336 payload_rows_remote=53162 dup_rows=4200 dup_rows_remote=4146 dedup_ratio=0.064087 router_tokens_per_expert=[0, 4093, 0, 0, 0, 3812, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2410, 0, 3989, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3861, 1762, 54, 0, 0, 0, 4096, 9, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7624, 8186, 4820, 7978, 8174, 8192, 8170, 8192] dup_rows_per_dst=[0, 938, 0, 0, 54, 0, 3202, 6] combine_rows_per_src=[7624, 8186, 4820, 7978, 8174, 8192, 8170, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=361496576 dispatch_meta_bytes=229232 combine_payload_bytes=361496576 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=361725808 ingress_bytes=691798016 link_bytes=691798016 token_bytes=723222384 weight_movement_bytes=330301440 total_nvlink_bytes=1053523824 weight_to_token_ratio=0.456708 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.203150531e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=364707840 dispatch_meta_bytes=219152 combine_payload_bytes=364707840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364926992 ingress_bytes=474808320 link_bytes=474808320 token_bytes=729634832 weight_movement_bytes=110100480 total_nvlink_bytes=839735312 weight_to_token_ratio=0.150898 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=18560 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.257697610e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58402 payload_rows_remote=50456 dup_rows=7134 dup_rows_remote=6986 dedup_ratio=0.108856 router_tokens_per_expert=[0, 0, 1765, 0, 0, 3826, 0, 0, 11, 0, 0, 68, 5, 0, 0, 0, 0, 0, 2659, 0, 0, 0, 3963, 0, 0, 3963, 0, 3818, 0, 1, 18, 0, 0, 0, 0, 0, 2729, 1248, 0, 1158, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 3211, 4083, 0] payload_rows_per_dst=[8128, 7946, 5318, 7816, 6454, 7928, 8166, 6646] dup_rows_per_dst=[3054, 148, 0, 1996, 1678, 12, 124, 122] combine_rows_per_src=[8128, 7946, 5318, 7816, 6454, 7928, 8166, 6646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=377194496 dispatch_meta_bytes=221768 combine_payload_bytes=377194496 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=377416264 ingress_bytes=927696896 link_bytes=927696896 token_bytes=754610760 weight_movement_bytes=550502400 total_nvlink_bytes=1305113160 weight_to_token_ratio=0.729518 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=14976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.613417482e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=344551424 dispatch_meta_bytes=216664 combine_payload_bytes=344551424 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=344768088 ingress_bytes=564752384 link_bytes=564752384 token_bytes=689319512 weight_movement_bytes=220200960 total_nvlink_bytes=909520472 weight_to_token_ratio=0.319447 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=15616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.821972816e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=360306688 dispatch_meta_bytes=229440 combine_payload_bytes=360306688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=360536128 ingress_bytes=470407168 link_bytes=470407168 token_bytes=720842816 weight_movement_bytes=110100480 total_nvlink_bytes=830943296 weight_to_token_ratio=0.152739 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.181154338e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=381065216 dispatch_meta_bytes=229232 combine_payload_bytes=381065216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=381294448 ingress_bytes=491165696 link_bytes=491165696 token_bytes=762359664 weight_movement_bytes=110100480 total_nvlink_bytes=872460144 weight_to_token_ratio=0.144421 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.542179282e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=361668608 dispatch_meta_bytes=229768 combine_payload_bytes=361668608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=361898376 ingress_bytes=471769088 link_bytes=471769088 token_bytes=723566984 weight_movement_bytes=110100480 total_nvlink_bytes=833667464 weight_to_token_ratio=0.152163 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.204840367e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60942 payload_rows_remote=52750 dup_rows=4594 dup_rows_remote=4590 dedup_ratio=0.070099 router_tokens_per_expert=[0, 4091, 0, 0, 0, 3579, 0, 0, 822, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2771, 0, 3995, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3903, 1150, 128, 0, 0, 0, 4096, 15, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6486, 8190, 5542, 7990, 8190, 8192, 8160, 8192] dup_rows_per_dst=[0, 1640, 0, 0, 46, 8, 2896, 4] combine_rows_per_src=[6486, 8190, 5542, 7990, 8190, 8192, 8160, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61390 payload_rows_remote=53204 dup_rows=4146 dup_rows_remote=4146 dedup_ratio=0.063263 router_tokens_per_expert=[0, 4087, 0, 0, 0, 3803, 0, 0, 247, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2389, 0, 4044, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3937, 1774, 157, 0, 0, 0, 4096, 11, 0, 8, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7606, 8184, 4778, 8088, 8180, 8186, 8176, 8192] dup_rows_per_dst=[0, 494, 0, 0, 54, 0, 3582, 16] combine_rows_per_src=[7606, 8184, 4778, 8088, 8180, 8186, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58832 payload_rows_remote=50640 dup_rows=6704 dup_rows_remote=2406 dedup_ratio=0.102295 router_tokens_per_expert=[0, 3943, 0, 2038, 0, 1368, 0, 1173, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 4084, 0, 0, 0, 0, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 3494, 0, 0, 0, 0, 0, 4040, 0, 6, 165, 0, 0, 4096, 0, 0, 0, 125, 0, 0, 1948, 0, 0, 0, 8, 31, 0, 0, 0] payload_rows_per_dst=[7138, 8192, 8168, 8192, 6988, 8094, 4136, 7924] dup_rows_per_dst=[2020, 0, 8, 4298, 0, 328, 10, 40] combine_rows_per_src=[7138, 8192, 8168, 8192, 6988, 8094, 4136, 7924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52944 payload_rows_remote=44872 dup_rows=12592 dup_rows_remote=6472 dedup_ratio=0.192139 router_tokens_per_expert=[0, 4096, 0, 0, 0, 3679, 0, 0, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3075, 0, 3964, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3899, 1319, 58, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8072, 1946, 7970, 7998, 7860, 8192, 2858, 8048] dup_rows_per_dst=[6120, 126, 5978, 256, 72, 0, 40, 0] combine_rows_per_src=[8072, 1946, 7970, 7998, 7860, 8192, 2858, 8048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59898 payload_rows_remote=51724 dup_rows=5638 dup_rows_remote=936 dedup_ratio=0.086029 router_tokens_per_expert=[0, 4094, 0, 0, 0, 3565, 0, 0, 447, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1937, 0, 3988, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3914, 2464, 56, 0, 0, 0, 4096, 4, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7130, 8190, 3874, 7976, 8170, 8192, 8174, 8192] dup_rows_per_dst=[0, 894, 0, 0, 36, 0, 4702, 6] combine_rows_per_src=[7130, 8190, 3874, 7976, 8170, 8192, 8174, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=378112000 dispatch_meta_bytes=229360 combine_payload_bytes=378112000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=378341360 ingress_bytes=488212480 link_bytes=488212480 token_bytes=756453360 weight_movement_bytes=110100480 total_nvlink_bytes=866553840 weight_to_token_ratio=0.145548 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.490817999e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64778 payload_rows_remote=57252 dup_rows=758 dup_rows_remote=758 dedup_ratio=0.011566 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 6, 0, 4096, 6, 4048, 0, 0, 0, 43, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4093, 0, 1, 0, 0, 0, 0, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 151, 0, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8106, 8192, 8186, 7526, 8192, 8192, 8192] dup_rows_per_dst=[222, 14, 86, 0, 0, 0, 432, 4] combine_rows_per_src=[8192, 8106, 8192, 8186, 7526, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60148 payload_rows_remote=52128 dup_rows=5388 dup_rows_remote=844 dedup_ratio=0.082214 router_tokens_per_expert=[0, 4088, 0, 0, 0, 3607, 0, 1, 387, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2366, 0, 3897, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3916, 2109, 75, 0, 0, 0, 4096, 20, 0, 9, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7214, 8190, 8020, 7794, 8156, 8190, 4392, 8192] dup_rows_per_dst=[2, 768, 4544, 0, 40, 0, 16, 18] combine_rows_per_src=[7214, 8190, 8020, 7794, 8156, 8190, 4392, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=381366272 dispatch_meta_bytes=229400 combine_payload_bytes=381366272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=381595672 ingress_bytes=491466752 link_bytes=491466752 token_bytes=762961944 weight_movement_bytes=110100480 total_nvlink_bytes=873062424 weight_to_token_ratio=0.144307 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.547415141e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=362987520 dispatch_meta_bytes=212184 combine_payload_bytes=362987520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=363199704 ingress_bytes=473088000 link_bytes=473088000 token_bytes=726187224 weight_movement_bytes=110100480 total_nvlink_bytes=836287704 weight_to_token_ratio=0.151614 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.227778416e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55058 payload_rows_remote=46934 dup_rows=10478 dup_rows_remote=10478 dedup_ratio=0.159882 router_tokens_per_expert=[0, 75, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3601, 0, 0, 0, 0, 0, 0, 1762, 290, 0, 0, 0, 0, 3953, 4050, 0, 4062, 0, 0, 0, 0, 2, 195, 3056, 0, 0, 3956, 0, 0, 0, 66, 0, 0, 0, 2065, 3172, 0, 0, 0, 910, 0, 0, 0, 1399, 0] payload_rows_per_dst=[8052, 8124, 7678, 8182, 8100, 2778, 7872, 4272] dup_rows_per_dst=[312, 0, 3126, 3834, 0, 126, 2734, 346] combine_rows_per_src=[8052, 8124, 7678, 8182, 8100, 2778, 7872, 4272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=321642496 dispatch_meta_bytes=205376 combine_payload_bytes=321642496 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=321847872 ingress_bytes=651943936 link_bytes=651943936 token_bytes=643490368 weight_movement_bytes=330301440 total_nvlink_bytes=973791808 weight_to_token_ratio=0.513297 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.133837731e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=370757632 dispatch_meta_bytes=210640 combine_payload_bytes=370757632 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=370968272 ingress_bytes=811159552 link_bytes=811159552 token_bytes=741725904 weight_movement_bytes=440401920 total_nvlink_bytes=1182127824 weight_to_token_ratio=0.593753 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.410739874e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=410382336 dispatch_meta_bytes=232040 combine_payload_bytes=410382336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410614376 ingress_bytes=520482816 link_bytes=520482816 token_bytes=820996712 weight_movement_bytes=110100480 total_nvlink_bytes=931097192 weight_to_token_ratio=0.134106 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.052052217e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=373653504 dispatch_meta_bytes=211888 combine_payload_bytes=373653504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373865392 ingress_bytes=483753984 link_bytes=483753984 token_bytes=747518896 weight_movement_bytes=110100480 total_nvlink_bytes=857619376 weight_to_token_ratio=0.147288 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=20224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.413277420e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=336422912 dispatch_meta_bytes=229648 combine_payload_bytes=336422912 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=336652560 ingress_bytes=556623872 link_bytes=556623872 token_bytes=673075472 weight_movement_bytes=220200960 total_nvlink_bytes=893276432 weight_to_token_ratio=0.327156 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.680604623e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57478 payload_rows_remote=49286 dup_rows=8058 dup_rows_remote=8054 dedup_ratio=0.122955 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 288, 0, 0, 11, 0, 4096, 31, 4051, 0, 0, 0, 56, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4088, 0, 0, 0, 0, 0, 0, 3448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 0, 137, 0, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[5432, 7090, 8192, 4008, 8180, 8192, 8192, 8192] dup_rows_per_dst=[366, 62, 112, 0, 6892, 0, 622, 4] combine_rows_per_src=[5432, 7090, 8192, 4008, 8180, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64042 payload_rows_remote=55850 dup_rows=1494 dup_rows_remote=1494 dedup_ratio=0.022797 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 11, 0, 4095, 90, 4053, 0, 0, 0, 31, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 4096, 4082, 0, 6, 0, 0, 0, 0, 3387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8134, 8192, 8164, 6786, 8192, 8192, 8192] dup_rows_per_dst=[442, 174, 62, 0, 0, 0, 816, 0] combine_rows_per_src=[8190, 8134, 8192, 8164, 6786, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57754 payload_rows_remote=49562 dup_rows=7782 dup_rows_remote=7278 dedup_ratio=0.118744 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 4096, 106, 4062, 0, 0, 0, 35, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4, 0, 0, 0, 0, 3377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8128, 7494, 8192, 6762, 8186, 2616, 8184] dup_rows_per_dst=[504, 908, 68, 6184, 0, 0, 118, 0] combine_rows_per_src=[8192, 8128, 7494, 8192, 6762, 8186, 2616, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59526 payload_rows_remote=51802 dup_rows=6010 dup_rows_remote=3662 dedup_ratio=0.091705 router_tokens_per_expert=[0, 4091, 0, 0, 0, 3781, 0, 0, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2762, 0, 3828, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3920, 1427, 123, 0, 0, 0, 4096, 37, 0, 4, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7562, 8186, 5968, 7724, 8192, 8192, 5510, 8192] dup_rows_per_dst=[0, 964, 1528, 2348, 46, 0, 1116, 8] combine_rows_per_src=[7562, 8186, 5968, 7724, 8192, 8192, 5510, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=353282048 dispatch_meta_bytes=229360 combine_payload_bytes=353282048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=353511408 ingress_bytes=463382528 link_bytes=463382528 token_bytes=706793456 weight_movement_bytes=110100480 total_nvlink_bytes=816893936 weight_to_token_ratio=0.155775 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.058984295e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63044 payload_rows_remote=54854 dup_rows=2492 dup_rows_remote=2076 dedup_ratio=0.038025 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 111, 0, 0, 3, 0, 4096, 63, 4045, 0, 0, 0, 43, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 4093, 0, 2, 0, 0, 0, 0, 3719, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 32, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8112, 8192, 6488, 7486, 8192, 8190, 8192] dup_rows_per_dst=[224, 110, 86, 0, 1654, 2, 416, 0] combine_rows_per_src=[8192, 8112, 8192, 6488, 7486, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58798 payload_rows_remote=51614 dup_rows=6738 dup_rows_remote=3798 dedup_ratio=0.102814 router_tokens_per_expert=[0, 0, 3120, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 500, 0, 1989, 2573, 0, 0, 3698, 0, 0, 0, 2359, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1940, 0, 0, 0, 0, 113, 4093, 0] payload_rows_per_dst=[6256, 8182, 8192, 8186, 7184, 4718, 8192, 7888] dup_rows_per_dst=[0, 6, 0, 178, 2940, 0, 0, 3614] combine_rows_per_src=[6256, 8182, 8192, 8186, 7184, 4718, 8192, 7888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64580 payload_rows_remote=56388 dup_rows=956 dup_rows_remote=884 dedup_ratio=0.014587 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 23, 0, 4095, 50, 4050, 0, 0, 0, 36, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 4096, 4059, 0, 1, 0, 0, 0, 0, 3688, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 50, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8126, 8192, 8118, 7378, 8192, 8192, 8192] dup_rows_per_dst=[254, 120, 72, 0, 0, 4, 506, 0] combine_rows_per_src=[8190, 8126, 8192, 8118, 7378, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=400332800 dispatch_meta_bytes=229376 combine_payload_bytes=400332800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400562176 ingress_bytes=510433280 link_bytes=510433280 token_bytes=800894976 weight_movement_bytes=110100480 total_nvlink_bytes=910995456 weight_to_token_ratio=0.137472 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.877274258e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58908 payload_rows_remote=50738 dup_rows=6628 dup_rows_remote=6628 dedup_ratio=0.101135 router_tokens_per_expert=[0, 3692, 0, 2162, 0, 1442, 0, 1490, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 36, 23, 0, 0, 4060, 0, 0, 0, 0, 1647, 0, 0, 0, 0, 0, 0, 0, 0, 3570, 1, 0, 0, 0, 0, 3792, 0, 80, 132, 0, 0, 4096, 0, 0, 0, 109, 0, 0, 2156, 0, 0, 0, 34, 161, 0, 0, 0] payload_rows_per_dst=[7488, 8170, 8130, 8192, 7142, 7748, 4504, 7534] dup_rows_per_dst=[2700, 0, 108, 3294, 0, 260, 26, 240] combine_rows_per_src=[7488, 8170, 8130, 8192, 7142, 7748, 4504, 7534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=355260416 dispatch_meta_bytes=227360 combine_payload_bytes=355260416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=355487776 ingress_bytes=465360896 link_bytes=465360896 token_bytes=710748192 weight_movement_bytes=110100480 total_nvlink_bytes=820848672 weight_to_token_ratio=0.154908 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.093391369e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=371316736 dispatch_meta_bytes=221856 combine_payload_bytes=371316736 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=371538592 ingress_bytes=591517696 link_bytes=591517696 token_bytes=742855328 weight_movement_bytes=220200960 total_nvlink_bytes=963056288 weight_to_token_ratio=0.296425 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.028746561e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=393193472 dispatch_meta_bytes=227720 combine_payload_bytes=393193472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=393421192 ingress_bytes=503293952 link_bytes=503293952 token_bytes=786614664 weight_movement_bytes=110100480 total_nvlink_bytes=896715144 weight_to_token_ratio=0.139967 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=29696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.753109602e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=369969152 dispatch_meta_bytes=221648 combine_payload_bytes=369969152 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=370190800 ingress_bytes=700270592 link_bytes=700270592 token_bytes=740159952 weight_movement_bytes=330301440 total_nvlink_bytes=1070461392 weight_to_token_ratio=0.446257 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=14848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.217885734e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=404189184 dispatch_meta_bytes=229088 combine_payload_bytes=404189184 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=404418272 ingress_bytes=514289664 link_bytes=514289664 token_bytes=808607456 weight_movement_bytes=110100480 total_nvlink_bytes=918707936 weight_to_token_ratio=0.136161 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.944343118e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=363689984 dispatch_meta_bytes=229464 combine_payload_bytes=363689984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=363919448 ingress_bytes=473790464 link_bytes=473790464 token_bytes=727609432 weight_movement_bytes=110100480 total_nvlink_bytes=837709912 weight_to_token_ratio=0.151318 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.239995420e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58442 payload_rows_remote=50502 dup_rows=7094 dup_rows_remote=1956 dedup_ratio=0.108246 router_tokens_per_expert=[0, 0, 3626, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 633, 1, 2567, 2818, 0, 0, 3508, 0, 0, 0, 831, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2190, 0, 0, 0, 0, 114, 4094, 0] payload_rows_per_dst=[7270, 8098, 8192, 6768, 6594, 5388, 8192, 7940] dup_rows_per_dst=[6, 8, 4, 136, 1142, 660, 0, 5138] combine_rows_per_src=[7270, 8098, 8192, 6768, 6594, 5388, 8192, 7940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60842 payload_rows_remote=54724 dup_rows=4694 dup_rows_remote=4694 dedup_ratio=0.071625 router_tokens_per_expert=[0, 0, 3480, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 4092, 14, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 556, 7, 1122, 2634, 0, 0, 3938, 0, 3, 0, 3056, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1385, 0, 0, 0, 0, 99, 4090, 0] payload_rows_per_dst=[6994, 8186, 8192, 8180, 6922, 6118, 8192, 8058] dup_rows_per_dst=[18, 26, 2, 146, 1716, 0, 0, 2786] combine_rows_per_src=[6994, 8186, 8192, 8180, 6922, 6118, 8192, 8058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=361998336 dispatch_meta_bytes=209832 combine_payload_bytes=361998336 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=362208168 ingress_bytes=582199296 link_bytes=582199296 token_bytes=724206504 weight_movement_bytes=220200960 total_nvlink_bytes=944407464 weight_to_token_ratio=0.304058 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=15616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.012540331e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57324 payload_rows_remote=49378 dup_rows=8212 dup_rows_remote=4118 dedup_ratio=0.125305 router_tokens_per_expert=[0, 0, 3616, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 589, 0, 1649, 2373, 0, 0, 3745, 0, 0, 0, 1946, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2189, 0, 0, 0, 0, 161, 4096, 0] payload_rows_per_dst=[7946, 8184, 8174, 8192, 4192, 7982, 8192, 4462] dup_rows_per_dst=[4094, 18, 0, 178, 284, 3400, 0, 238] combine_rows_per_src=[7946, 8184, 8174, 8192, 4192, 7982, 8192, 4462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63770 payload_rows_remote=55616 dup_rows=1766 dup_rows_remote=1766 dedup_ratio=0.026947 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 15, 0, 4096, 103, 4024, 0, 0, 0, 55, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4077, 0, 3, 0, 0, 0, 0, 3279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8092, 8192, 8154, 6564, 8192, 8192, 8192] dup_rows_per_dst=[356, 192, 110, 0, 0, 0, 1108, 0] combine_rows_per_src=[8192, 8092, 8192, 8154, 6564, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64712 payload_rows_remote=56520 dup_rows=824 dup_rows_remote=824 dedup_ratio=0.012573 router_tokens_per_expert=[16, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 386, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 3760, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 10, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8048, 7520, 8192, 8190, 8186, 8192] dup_rows_per_dst=[18, 772, 0, 4, 0, 10, 0, 20] combine_rows_per_src=[8192, 8192, 8048, 7520, 8192, 8190, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=392261632 dispatch_meta_bytes=237672 combine_payload_bytes=392261632 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=392499304 ingress_bytes=612462592 link_bytes=612462592 token_bytes=784760936 weight_movement_bytes=220200960 total_nvlink_bytes=1004961896 weight_to_token_ratio=0.280596 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.065173180e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59858 payload_rows_remote=51666 dup_rows=5678 dup_rows_remote=5678 dedup_ratio=0.086639 router_tokens_per_expert=[0, 0, 3730, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 1, 0, 0, 605, 3, 1593, 2817, 0, 0, 3340, 0, 0, 0, 2375, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1689, 0, 0, 0, 0, 132, 4095, 0] payload_rows_per_dst=[7476, 8192, 8192, 8190, 7252, 4750, 8192, 7614] dup_rows_per_dst=[4, 6, 0, 174, 2786, 0, 0, 2708] combine_rows_per_src=[7476, 8192, 8192, 8190, 7252, 4750, 8192, 7614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58318 payload_rows_remote=50126 dup_rows=7218 dup_rows_remote=7216 dedup_ratio=0.110138 router_tokens_per_expert=[0, 0, 3816, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 4086, 10, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 587, 4, 1389, 2371, 0, 0, 3751, 0, 6, 0, 2448, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1633, 0, 0, 0, 0, 214, 4092, 0] payload_rows_per_dst=[7776, 8174, 8192, 8184, 6206, 8052, 8192, 3542] dup_rows_per_dst=[894, 18, 2, 274, 1520, 4358, 0, 152] combine_rows_per_src=[7776, 8174, 8192, 8184, 6206, 8052, 8192, 3542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60268 payload_rows_remote=52156 dup_rows=5268 dup_rows_remote=4740 dedup_ratio=0.080383 router_tokens_per_expert=[0, 4044, 0, 0, 0, 3860, 0, 0, 264, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2480, 0, 3692, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3777, 2021, 89, 0, 0, 0, 4096, 161, 0, 43, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7720, 8112, 7996, 7384, 8190, 8192, 4482, 8192] dup_rows_per_dst=[0, 528, 4518, 0, 76, 0, 60, 86] combine_rows_per_src=[7720, 8112, 7996, 7384, 8190, 8192, 4482, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=353941504 dispatch_meta_bytes=213984 combine_payload_bytes=353941504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=354155488 ingress_bytes=464041984 link_bytes=464041984 token_bytes=708096992 weight_movement_bytes=110100480 total_nvlink_bytes=818197472 weight_to_token_ratio=0.155488 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.070453320e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=398655488 dispatch_meta_bytes=229528 combine_payload_bytes=398655488 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=398885016 ingress_bytes=949157888 link_bytes=949157888 token_bytes=797540504 weight_movement_bytes=550502400 total_nvlink_bytes=1348042904 weight_to_token_ratio=0.690250 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.650741677e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=405135360 dispatch_meta_bytes=229376 combine_payload_bytes=405135360 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=405364736 ingress_bytes=735436800 link_bytes=735436800 token_bytes=810500096 weight_movement_bytes=330301440 total_nvlink_bytes=1140801536 weight_to_token_ratio=0.407528 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.279045554e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61478 payload_rows_remote=53286 dup_rows=4058 dup_rows_remote=3982 dedup_ratio=0.061920 router_tokens_per_expert=[17, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 3, 0, 254, 0, 0, 0, 0, 0, 0, 2, 3988, 0, 0, 3911, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 4, 0, 0, 2, 0, 2, 0, 0, 4083, 0, 0, 38, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8018, 4704, 7980, 8038, 8188, 8188, 8170, 8192] dup_rows_per_dst=[14, 270, 0, 3512, 0, 8, 178, 76] combine_rows_per_src=[8018, 4704, 7980, 8038, 8188, 8188, 8170, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=370341888 dispatch_meta_bytes=229376 combine_payload_bytes=370341888 prefetch_bytes=0 reduce_bytes=0 egress_bytes=370571264 ingress_bytes=370341888 link_bytes=370571264 token_bytes=740913152 weight_movement_bytes=0 total_nvlink_bytes=740913152 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.444843766e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=359303168 dispatch_meta_bytes=229368 combine_payload_bytes=359303168 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=359532536 ingress_bytes=469403648 link_bytes=469403648 token_bytes=718835704 weight_movement_bytes=110100480 total_nvlink_bytes=828936184 weight_to_token_ratio=0.153165 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.163701475e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=373854208 dispatch_meta_bytes=227584 combine_payload_bytes=373854208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374081792 ingress_bytes=483954688 link_bytes=483954688 token_bytes=747936000 weight_movement_bytes=110100480 total_nvlink_bytes=858036480 weight_to_token_ratio=0.147206 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.416767993e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7283314240 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=3642801728 ingress_bytes=5842522112 link_bytes=5842522112 total_nvlink_bytes=9485323840 weight_to_token_ratio=0.302336 prefetch_share_of_ingress=0.169602 dedup_ratio=0.113177 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047794 gemm_inner_pad_upper_bound=6985 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.016110688e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65090 payload_rows_remote=56908 dup_rows=446 dup_rows_remote=440 dedup_ratio=0.006805 router_tokens_per_expert=[29, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 8, 0, 187, 0, 0, 0, 0, 0, 0, 5, 4012, 0, 0, 3970, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4091, 0, 0, 0, 4091, 0, 0, 3, 0, 0, 5, 0, 5, 0, 0, 4084, 0, 0, 4, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8192, 8192, 8034, 7940, 8182, 8182, 8178, 8190] dup_rows_per_dst=[38, 380, 0, 0, 0, 6, 10, 12] combine_rows_per_src=[8192, 8192, 8034, 7940, 8182, 8182, 8178, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=381954048 dispatch_meta_bytes=229072 combine_payload_bytes=381954048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382183120 ingress_bytes=492054528 link_bytes=492054528 token_bytes=764137168 weight_movement_bytes=110100480 total_nvlink_bytes=874237648 weight_to_token_ratio=0.144085 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.557637533e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64020 payload_rows_remote=55828 dup_rows=1516 dup_rows_remote=1508 dedup_ratio=0.023132 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4009, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8174, 7092, 8192, 8172, 8190, 7816] dup_rows_per_dst=[8, 196, 1218, 0, 94, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8174, 7092, 8192, 8172, 8190, 7816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58790 payload_rows_remote=50604 dup_rows=6746 dup_rows_remote=6558 dedup_ratio=0.102936 router_tokens_per_expert=[0, 0, 3377, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 4087, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 1031, 4, 2125, 2080, 0, 0, 3608, 0, 0, 0, 2001, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1922, 0, 0, 0, 0, 117, 4093, 0] payload_rows_per_dst=[6788, 8174, 8192, 8186, 7456, 6646, 8192, 5156] dup_rows_per_dst=[14, 26, 0, 188, 3024, 2426, 0, 1068] combine_rows_per_src=[6788, 8174, 8192, 8186, 7456, 6646, 8192, 5156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6705812336 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=3353998192 ingress_bytes=5113421824 link_bytes=5113421824 total_nvlink_bytes=8467420016 weight_to_token_ratio=0.262699 prefetch_share_of_ingress=0.215317 dedup_ratio=0.168027 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048897 gemm_inner_pad_upper_bound=7366 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.893081565e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=407916544 dispatch_meta_bytes=229392 combine_payload_bytes=407916544 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=408145936 ingress_bytes=628117504 link_bytes=628117504 token_bytes=816062480 weight_movement_bytes=220200960 total_nvlink_bytes=1036263440 weight_to_token_ratio=0.269833 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.092399647e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65218 payload_rows_remote=57026 dup_rows=318 dup_rows_remote=316 dedup_ratio=0.004852 router_tokens_per_expert=[6, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 1, 0, 145, 0, 0, 0, 0, 0, 0, 2, 4042, 0, 0, 3993, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 4094, 0, 0, 7, 0, 0, 1, 0, 1, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8088, 7986, 8190, 8188, 8192, 8190] dup_rows_per_dst=[6, 290, 0, 2, 0, 14, 2, 4] combine_rows_per_src=[8192, 8192, 8088, 7986, 8190, 8188, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65072 payload_rows_remote=57008 dup_rows=464 dup_rows_remote=464 dedup_ratio=0.007080 router_tokens_per_expert=[34, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 9, 0, 185, 0, 0, 0, 0, 0, 0, 4, 4028, 0, 0, 3963, 4092, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 4, 0, 3, 0, 0, 4075, 0, 0, 28, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8182, 8192, 8064, 7926, 8174, 8184, 8158, 8192] dup_rows_per_dst=[6, 380, 0, 14, 0, 2, 6, 56] combine_rows_per_src=[8182, 8192, 8064, 7926, 8174, 8184, 8158, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=400175104 dispatch_meta_bytes=229344 combine_payload_bytes=400175104 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=400404448 ingress_bytes=620376064 link_bytes=620376064 token_bytes=800579552 weight_movement_bytes=220200960 total_nvlink_bytes=1020780512 weight_to_token_ratio=0.275052 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.078936010e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7201441824 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=3601858592 ingress_bytes=5691492352 link_bytes=5691492352 total_nvlink_bytes=9293350944 weight_to_token_ratio=0.290485 prefetch_share_of_ingress=0.174103 dedup_ratio=0.118137 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047426 gemm_inner_pad_upper_bound=7493 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.898441290e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63322 payload_rows_remote=55308 dup_rows=2214 dup_rows_remote=2060 dedup_ratio=0.033783 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 0, 197, 0, 0, 72, 0, 4094, 95, 3917, 0, 0, 0, 51, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4082, 4096, 4021, 0, 20, 0, 0, 0, 0, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 0, 0, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8014, 8192, 8042, 6308, 8192, 8192, 8192] dup_rows_per_dst=[398, 154, 102, 0, 0, 0, 1560, 0] combine_rows_per_src=[8190, 8014, 8192, 8042, 6308, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=362729472 dispatch_meta_bytes=228648 combine_payload_bytes=362729472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362958120 ingress_bytes=472829952 link_bytes=472829952 token_bytes=725687592 weight_movement_bytes=110100480 total_nvlink_bytes=835788072 weight_to_token_ratio=0.151719 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.223290537e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=408762368 dispatch_meta_bytes=229368 combine_payload_bytes=408762368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=408991736 ingress_bytes=518862848 link_bytes=518862848 token_bytes=817754104 weight_movement_bytes=110100480 total_nvlink_bytes=927854584 weight_to_token_ratio=0.134638 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.023878309e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6697715200 prefetch_bytes=880803840 reduce_bytes=1871708160 weight_movement_bytes=2752512000 egress_bytes=3349943808 ingress_bytes=6100283392 link_bytes=6100283392 total_nvlink_bytes=9450227200 weight_to_token_ratio=0.410963 prefetch_share_of_ingress=0.144387 dedup_ratio=0.167413 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=17 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.060939614e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=408633344 dispatch_meta_bytes=229888 combine_payload_bytes=408633344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=408863232 ingress_bytes=518733824 link_bytes=518733824 token_bytes=817496576 weight_movement_bytes=110100480 total_nvlink_bytes=927597056 weight_to_token_ratio=0.134680 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.021634370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=396447744 dispatch_meta_bytes=229472 combine_payload_bytes=396447744 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=396677216 ingress_bytes=726749184 link_bytes=726749184 token_bytes=793124960 weight_movement_bytes=330301440 total_nvlink_bytes=1123426400 weight_to_token_ratio=0.416456 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.263936361e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6986164840 prefetch_bytes=990904320 reduce_bytes=660602880 weight_movement_bytes=1651507200 egress_bytes=3494216296 ingress_bytes=5143455744 link_bytes=5143685120 total_nvlink_bytes=8637672040 weight_to_token_ratio=0.236397 prefetch_share_of_ingress=0.192653 dedup_ratio=0.144153 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=6 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048529 gemm_inner_pad_upper_bound=7366 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.945714415e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7308415376 prefetch_bytes=880803840 reduce_bytes=660602880 weight_movement_bytes=1541406720 egress_bytes=3655358864 ingress_bytes=5194463232 link_bytes=5194463232 total_nvlink_bytes=8849822096 weight_to_token_ratio=0.210908 prefetch_share_of_ingress=0.169566 dedup_ratio=0.113190 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=6 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048897 gemm_inner_pad_upper_bound=6858 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.034025903e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64948 payload_rows_remote=57188 dup_rows=588 dup_rows_remote=588 dedup_ratio=0.008972 router_tokens_per_expert=[13, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, 1, 4021, 0, 0, 3880, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8044, 7760, 8192, 8188, 8192, 8192] dup_rows_per_dst=[4, 538, 0, 0, 0, 10, 0, 36] combine_rows_per_src=[8188, 8192, 8044, 7760, 8192, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=409923584 dispatch_meta_bytes=231104 combine_payload_bytes=409923584 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=410154688 ingress_bytes=740225024 link_bytes=740225024 token_bytes=820078272 weight_movement_bytes=330301440 total_nvlink_bytes=1150379712 weight_to_token_ratio=0.402768 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.287373063e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58884 payload_rows_remote=50752 dup_rows=6652 dup_rows_remote=6546 dedup_ratio=0.101501 router_tokens_per_expert=[0, 0, 3026, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 4066, 53, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 669, 52, 2438, 1915, 0, 0, 3653, 0, 17, 0, 1849, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2512, 0, 0, 0, 0, 74, 4067, 0] payload_rows_per_dst=[7390, 8132, 8192, 8136, 5828, 7982, 8192, 5032] dup_rows_per_dst=[2610, 106, 10, 240, 490, 3056, 0, 140] combine_rows_per_src=[7390, 8132, 8192, 8136, 5828, 7982, 8192, 5032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=363790336 dispatch_meta_bytes=229192 combine_payload_bytes=363790336 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=364019528 ingress_bytes=583991296 link_bytes=583991296 token_bytes=727809864 weight_movement_bytes=220200960 total_nvlink_bytes=948010824 weight_to_token_ratio=0.302553 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.015656914e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7215881232 prefetch_bytes=990904320 reduce_bytes=1321205760 weight_movement_bytes=2312110080 egress_bytes=3609072656 ingress_bytes=5918918656 link_bytes=5918918656 total_nvlink_bytes=9527991312 weight_to_token_ratio=0.320420 prefetch_share_of_ingress=0.167413 dedup_ratio=0.118430 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=12 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047794 gemm_inner_pad_upper_bound=6858 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.029397304e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64562 payload_rows_remote=56370 dup_rows=974 dup_rows_remote=312 dedup_ratio=0.014862 router_tokens_per_expert=[103, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 38, 0, 308, 0, 0, 0, 0, 0, 0, 12, 3931, 0, 0, 3860, 4081, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 4066, 17, 0, 0, 4078, 0, 0, 2, 0, 0, 6, 0, 25, 0, 0, 4006, 0, 0, 88, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7886, 7720, 8132, 8188, 8060, 8192] dup_rows_per_dst=[94, 662, 0, 22, 0, 6, 14, 176] combine_rows_per_src=[8192, 8192, 7886, 7720, 8132, 8188, 8060, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=404060160 dispatch_meta_bytes=226728 combine_payload_bytes=404060160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404286888 ingress_bytes=514160640 link_bytes=514160640 token_bytes=808347048 weight_movement_bytes=110100480 total_nvlink_bytes=918447528 weight_to_token_ratio=0.136204 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.942099179e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6983952784 prefetch_bytes=1211105280 reduce_bytes=550502400 weight_movement_bytes=1761607680 egress_bytes=3493108112 ingress_bytes=5252452352 link_bytes=5252452352 total_nvlink_bytes=8745560464 weight_to_token_ratio=0.252236 prefetch_share_of_ingress=0.230579 dedup_ratio=0.138214 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=5 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=6731 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.134878520e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none + [2026-08-26 18:09:54] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 368.2 | throughput per GPU (TFLOP/s/GPU): 335.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.602648E+00 | loss scale: 1.0 | grad norm: 4.396 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44880 payload_rows_remote=39014 dup_rows=20656 dup_rows_remote=17392 dedup_ratio=0.315186 router_tokens_per_expert=[1222, 1028, 381, 864, 324, 599, 619, 28, 271, 846, 635, 305, 15, 941, 31, 963, 533, 782, 3, 29, 841, 221, 857, 427, 285, 582, 372, 762, 726, 279, 802, 5, 571, 577, 430, 3, 997, 0, 1041, 534, 718, 666, 313, 1519, 1, 773, 149, 426, 463, 365, 530, 428, 310, 760, 72, 393, 462, 1396, 655, 263, 473, 603, 295, 4] payload_rows_per_dst=[4974, 5500, 5722, 5372, 5348, 5866, 5950, 6148] dup_rows_per_dst=[1402, 2514, 2080, 2254, 2958, 3264, 3136, 3048] combine_rows_per_src=[4974, 5500, 5722, 5372, 5348, 5866, 5950, 6148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43860 payload_rows_remote=37946 dup_rows=21676 dup_rows_remote=17786 dedup_ratio=0.330750 router_tokens_per_expert=[1134, 315, 590, 340, 519, 1083, 826, 75, 224, 950, 692, 531, 12, 1153, 118, 630, 934, 467, 16, 17, 318, 426, 672, 937, 277, 438, 515, 1114, 378, 598, 593, 21, 861, 338, 445, 14, 800, 0, 1068, 367, 614, 571, 280, 650, 2, 722, 503, 457, 454, 357, 353, 400, 387, 1606, 79, 657, 535, 944, 565, 175, 228, 1066, 355, 2] payload_rows_per_dst=[5914, 5506, 5298, 5726, 5136, 5166, 5808, 5306] dup_rows_per_dst=[3890, 3114, 2276, 2142, 2610, 2432, 2778, 2434] combine_rows_per_src=[5914, 5506, 5298, 5726, 5136, 5166, 5808, 5306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38202 dup_rows=21166 dup_rows_remote=18074 dedup_ratio=0.322968 router_tokens_per_expert=[762, 1532, 1160, 516, 528, 1091, 874, 65, 781, 944, 605, 311, 31, 984, 181, 785, 473, 505, 9, 73, 741, 232, 601, 310, 367, 692, 461, 740, 609, 358, 530, 17, 857, 909, 526, 4, 716, 4, 1148, 431, 277, 701, 295, 1235, 1, 370, 151, 431, 219, 366, 573, 493, 221, 666, 66, 701, 920, 551, 453, 266, 507, 459, 379, 4] payload_rows_per_dst=[4774, 5392, 5952, 6092, 4840, 5682, 5470, 6168] dup_rows_per_dst=[1512, 1964, 3000, 3344, 2054, 3536, 2664, 3092] combine_rows_per_src=[4774, 5392, 5952, 6092, 4840, 5682, 5470, 6168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44312 payload_rows_remote=38814 dup_rows=21224 dup_rows_remote=18382 dedup_ratio=0.323853 router_tokens_per_expert=[1489, 526, 624, 245, 506, 517, 568, 222, 298, 1032, 416, 695, 8, 444, 69, 826, 541, 1109, 46, 46, 630, 204, 825, 539, 680, 551, 356, 830, 416, 307, 755, 24, 953, 607, 358, 25, 733, 12, 1194, 288, 529, 762, 260, 983, 4, 648, 270, 719, 236, 428, 471, 681, 314, 903, 123, 700, 572, 957, 439, 368, 429, 1082, 363, 13] payload_rows_per_dst=[5620, 5380, 5704, 5548, 5498, 5498, 5484, 5580] dup_rows_per_dst=[2716, 2724, 2176, 2290, 2842, 2852, 2758, 2866] combine_rows_per_src=[5620, 5380, 5704, 5548, 5498, 5498, 5484, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43912 payload_rows_remote=38268 dup_rows=21624 dup_rows_remote=19276 dedup_ratio=0.329956 router_tokens_per_expert=[816, 796, 788, 607, 512, 757, 608, 175, 728, 681, 393, 406, 14, 1011, 247, 753, 587, 852, 191, 102, 795, 365, 239, 865, 381, 1432, 631, 716, 1075, 415, 894, 12, 211, 642, 408, 14, 936, 8, 1244, 365, 243, 628, 719, 1145, 3, 479, 192, 751, 261, 250, 121, 746, 222, 538, 190, 679, 610, 540, 261, 331, 301, 432, 437, 17] payload_rows_per_dst=[6120, 5526, 5644, 6612, 5258, 5706, 4558, 4488] dup_rows_per_dst=[3998, 2940, 2348, 4500, 2398, 2614, 1456, 1370] combine_rows_per_src=[6120, 5526, 5644, 6612, 5258, 5706, 4558, 4488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44074 payload_rows_remote=38910 dup_rows=21462 dup_rows_remote=19904 dedup_ratio=0.327484 router_tokens_per_expert=[1281, 891, 483, 656, 487, 762, 450, 264, 406, 832, 506, 515, 16, 770, 96, 959, 475, 1179, 10, 94, 1006, 390, 708, 406, 591, 422, 365, 638, 303, 380, 650, 12, 789, 527, 573, 12, 1083, 3, 940, 535, 355, 572, 474, 1200, 8, 488, 51, 388, 259, 187, 697, 615, 464, 796, 225, 528, 369, 1046, 602, 319, 351, 886, 390, 33] payload_rows_per_dst=[6378, 5402, 5982, 5164, 5638, 5102, 5198, 5210] dup_rows_per_dst=[4170, 2798, 2554, 1558, 3286, 1970, 2344, 2782] combine_rows_per_src=[6378, 5402, 5982, 5164, 5638, 5102, 5198, 5210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44048 payload_rows_remote=37828 dup_rows=21488 dup_rows_remote=17524 dedup_ratio=0.327881 router_tokens_per_expert=[1068, 1345, 881, 289, 344, 917, 532, 135, 666, 1011, 288, 503, 16, 896, 163, 679, 341, 802, 3, 72, 601, 469, 732, 449, 447, 286, 601, 538, 362, 450, 798, 13, 997, 1060, 529, 41, 376, 13, 699, 820, 713, 510, 278, 751, 0, 562, 149, 613, 627, 175, 480, 949, 239, 755, 104, 695, 574, 693, 611, 676, 326, 839, 207, 10] payload_rows_per_dst=[3476, 5254, 6184, 5624, 5672, 5130, 6220, 6488] dup_rows_per_dst=[886, 2416, 3444, 2140, 2736, 2684, 3964, 3218] combine_rows_per_src=[3476, 5254, 6184, 5624, 5672, 5130, 6220, 6488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=225624 combine_payload_bytes=279652352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279877976 ingress_bytes=389752832 link_bytes=389752832 token_bytes=559530328 weight_movement_bytes=110100480 total_nvlink_bytes=669630808 weight_to_token_ratio=0.196773 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778442782e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=271996928 dispatch_meta_bytes=222928 combine_payload_bytes=271996928 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=272219856 ingress_bytes=822499328 link_bytes=822499328 token_bytes=544216784 weight_movement_bytes=550502400 total_nvlink_bytes=1094719184 weight_to_token_ratio=1.011550 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=5504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.430461609e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=273831936 dispatch_meta_bytes=225104 combine_payload_bytes=273831936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=274057040 ingress_bytes=383932416 link_bytes=383932416 token_bytes=547888976 weight_movement_bytes=110100480 total_nvlink_bytes=657989456 weight_to_token_ratio=0.200954 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.677216175e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278218752 dispatch_meta_bytes=228784 combine_payload_bytes=278218752 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278447536 ingress_bytes=498419712 link_bytes=498419712 token_bytes=556666288 weight_movement_bytes=220200960 total_nvlink_bytes=776867248 weight_to_token_ratio=0.395571 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.668338551e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44804 payload_rows_remote=38744 dup_rows=20732 dup_rows_remote=17686 dedup_ratio=0.316345 router_tokens_per_expert=[1292, 794, 575, 286, 488, 550, 617, 251, 285, 1241, 898, 333, 9, 815, 208, 764, 717, 725, 49, 131, 763, 253, 406, 542, 523, 627, 400, 683, 590, 268, 457, 37, 778, 692, 1120, 17, 806, 16, 942, 260, 595, 904, 373, 773, 1, 758, 117, 496, 366, 199, 407, 576, 208, 1066, 155, 897, 427, 1289, 590, 249, 487, 271, 344, 12] payload_rows_per_dst=[6222, 6060, 5388, 5176, 5790, 5348, 5496, 5324] dup_rows_per_dst=[3484, 3046, 1784, 1994, 3472, 2686, 2252, 2014] combine_rows_per_src=[6222, 6060, 5388, 5176, 5790, 5348, 5496, 5324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=274305024 dispatch_meta_bytes=230176 combine_payload_bytes=274305024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=274535200 ingress_bytes=384405504 link_bytes=384405504 token_bytes=548840224 weight_movement_bytes=110100480 total_nvlink_bytes=658940704 weight_to_token_ratio=0.200606 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.685443953e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278906880 dispatch_meta_bytes=235256 combine_payload_bytes=278906880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279142136 ingress_bytes=389007360 link_bytes=389007360 token_bytes=558049016 weight_movement_bytes=110100480 total_nvlink_bytes=668149496 weight_to_token_ratio=0.197295 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765477798e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=271151104 dispatch_meta_bytes=221408 combine_payload_bytes=271151104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=271372512 ingress_bytes=381251584 link_bytes=381251584 token_bytes=542523616 weight_movement_bytes=110100480 total_nvlink_bytes=652624096 weight_to_token_ratio=0.202941 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.630592097e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47676 payload_rows_remote=39592 dup_rows=17860 dup_rows_remote=12890 dedup_ratio=0.272522 router_tokens_per_expert=[0, 0, 0, 102, 1112, 0, 2220, 145, 920, 2116, 58, 0, 0, 3153, 0, 3326, 0, 2067, 0, 0, 405, 3932, 0, 0, 0, 0, 823, 46, 1245, 15, 0, 13, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 441, 0, 0, 1, 2153, 0, 0, 2204, 446, 0, 2038, 0, 0, 0, 1997, 3, 1, 53, 51, 508, 322, 845] payload_rows_per_dst=[6154, 2376, 4730, 5952, 7908, 8084, 6466, 6006] dup_rows_per_dst=[1004, 70, 214, 2564, 4574, 4970, 2910, 1554] combine_rows_per_src=[6154, 2376, 4730, 5952, 7908, 8084, 6466, 6006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277716992 dispatch_meta_bytes=225720 combine_payload_bytes=277716992 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277942712 ingress_bytes=497917952 link_bytes=497917952 token_bytes=555659704 weight_movement_bytes=220200960 total_nvlink_bytes=775860664 weight_to_token_ratio=0.396287 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.659612120e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46826 payload_rows_remote=38762 dup_rows=18710 dup_rows_remote=12606 dedup_ratio=0.285492 router_tokens_per_expert=[0, 0, 0, 59, 1629, 0, 2493, 136, 1091, 3005, 49, 0, 1, 2336, 0, 3543, 4, 1877, 0, 0, 418, 3849, 0, 0, 0, 0, 1179, 94, 1643, 89, 1, 4, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 334, 0, 0, 4, 1811, 0, 0, 2067, 354, 0, 1459, 0, 0, 2, 1682, 0, 0, 29, 13, 242, 411, 858] payload_rows_per_dst=[7112, 2228, 4412, 7218, 7784, 4080, 5928, 8064] dup_rows_per_dst=[1522, 54, 186, 4812, 3978, 218, 1836, 6104] combine_rows_per_src=[7112, 2228, 4412, 7218, 7784, 4080, 5928, 8064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47882 payload_rows_remote=40968 dup_rows=17654 dup_rows_remote=15514 dedup_ratio=0.269379 router_tokens_per_expert=[0, 0, 0, 13, 1462, 0, 1961, 163, 1653, 2156, 118, 0, 0, 2209, 0, 2713, 3, 1911, 0, 0, 425, 3906, 0, 0, 0, 0, 1520, 50, 1517, 53, 0, 5, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1291, 0, 0, 0, 2024, 0, 0, 1977, 473, 0, 1358, 0, 0, 0, 2082, 0, 0, 65, 27, 498, 291, 834] payload_rows_per_dst=[6914, 7802, 8018, 4892, 5432, 5384, 3354, 6086] dup_rows_per_dst=[2140, 8190, 2850, 1398, 14, 1246, 308, 1508] combine_rows_per_src=[6914, 7802, 8018, 4892, 5432, 5384, 3354, 6086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49298 payload_rows_remote=43116 dup_rows=16238 dup_rows_remote=16218 dedup_ratio=0.247772 router_tokens_per_expert=[0, 0, 0, 82, 1546, 1, 2313, 217, 2085, 1692, 168, 0, 0, 1625, 0, 3085, 0, 1475, 0, 1, 610, 3857, 0, 0, 1, 0, 1511, 34, 1431, 56, 1, 0, 0, 0, 0, 1, 0, 14, 0, 1, 0, 3, 706, 2, 0, 1, 1948, 0, 0, 2164, 390, 0, 1896, 0, 0, 0, 1768, 0, 0, 52, 18, 443, 468, 1102] payload_rows_per_dst=[6822, 5742, 3900, 6126, 6182, 8060, 6338, 6128] dup_rows_per_dst=[1496, 2014, 272, 3326, 20, 4974, 2562, 1574] combine_rows_per_src=[6822, 5742, 3900, 6126, 6182, 8060, 6338, 6128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49038 payload_rows_remote=40908 dup_rows=16498 dup_rows_remote=10830 dedup_ratio=0.251740 router_tokens_per_expert=[1, 0, 0, 38, 959, 3, 2536, 89, 1374, 2080, 175, 0, 0, 1918, 0, 3437, 2, 2250, 0, 0, 788, 3859, 0, 0, 0, 1, 1337, 47, 1065, 151, 0, 16, 1, 8, 0, 0, 0, 12, 0, 0, 0, 1, 1191, 0, 3, 0, 1285, 1, 0, 2103, 608, 0, 2353, 0, 0, 0, 1224, 0, 0, 36, 20, 571, 527, 698] payload_rows_per_dst=[6540, 7058, 8130, 4174, 6890, 4398, 6696, 5152] dup_rows_per_dst=[712, 4036, 5668, 1060, 26, 564, 3432, 1000] combine_rows_per_src=[6540, 7058, 8130, 4174, 6890, 4398, 6696, 5152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49278 payload_rows_remote=43992 dup_rows=16258 dup_rows_remote=14052 dedup_ratio=0.248077 router_tokens_per_expert=[0, 0, 0, 118, 1611, 0, 1971, 181, 1617, 1817, 151, 0, 3, 1669, 0, 3163, 0, 2225, 0, 0, 671, 3590, 0, 0, 0, 0, 1041, 195, 1723, 64, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 786, 1, 0, 0, 1665, 0, 0, 2047, 561, 0, 2014, 0, 0, 0, 1657, 0, 1, 42, 18, 701, 313, 1147] payload_rows_per_dst=[6430, 6316, 7408, 5286, 6326, 5068, 6324, 6120] dup_rows_per_dst=[1332, 2758, 4340, 2206, 2, 1062, 2920, 1638] combine_rows_per_src=[6430, 6316, 7408, 5286, 6326, 5068, 6324, 6120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=283795456 dispatch_meta_bytes=209928 combine_payload_bytes=283795456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=284005384 ingress_bytes=393895936 link_bytes=393895936 token_bytes=567800840 weight_movement_bytes=110100480 total_nvlink_bytes=677901320 weight_to_token_ratio=0.193907 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=14720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.850498175e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277846016 dispatch_meta_bytes=205472 combine_payload_bytes=277846016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278051488 ingress_bytes=387946496 link_bytes=387946496 token_bytes=555897504 weight_movement_bytes=110100480 total_nvlink_bytes=665997984 weight_to_token_ratio=0.198059 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747027628e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=293658624 dispatch_meta_bytes=225928 combine_payload_bytes=293658624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293884552 ingress_bytes=403759104 link_bytes=403759104 token_bytes=587543176 weight_movement_bytes=110100480 total_nvlink_bytes=697643656 weight_to_token_ratio=0.187391 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=18432 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.022034888e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48438 payload_rows_remote=42118 dup_rows=17098 dup_rows_remote=14634 dedup_ratio=0.260895 router_tokens_per_expert=[0, 0, 0, 15, 1960, 0, 2849, 125, 1559, 2927, 152, 0, 0, 2180, 0, 2906, 0, 1683, 0, 0, 326, 3788, 0, 0, 0, 0, 565, 37, 1252, 82, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 1757, 0, 0, 2191, 464, 0, 1736, 0, 0, 1, 1779, 0, 0, 37, 64, 373, 547, 783] payload_rows_per_dst=[7606, 3196, 3918, 6546, 7126, 7858, 6320, 5868] dup_rows_per_dst=[2292, 226, 100, 3186, 3046, 4194, 2464, 1590] combine_rows_per_src=[7606, 3196, 3918, 6546, 7126, 7858, 6320, 5868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=309055488 dispatch_meta_bytes=237336 combine_payload_bytes=309055488 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=309292824 ingress_bytes=529256448 link_bytes=529256448 token_bytes=618348312 weight_movement_bytes=220200960 total_nvlink_bytes=838549272 weight_to_token_ratio=0.356112 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=25344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.204640108e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=293228544 dispatch_meta_bytes=206952 combine_payload_bytes=293228544 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=293435496 ingress_bytes=733630464 link_bytes=733630464 token_bytes=586664040 weight_movement_bytes=440401920 total_nvlink_bytes=1027065960 weight_to_token_ratio=0.750688 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.275904038e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=315334656 dispatch_meta_bytes=232176 combine_payload_bytes=315334656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=315566832 ingress_bytes=425435136 link_bytes=425435136 token_bytes=630901488 weight_movement_bytes=110100480 total_nvlink_bytes=741001968 weight_to_token_ratio=0.174513 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.399016736e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=301901824 dispatch_meta_bytes=227008 combine_payload_bytes=301901824 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=302128832 ingress_bytes=522102784 link_bytes=522102784 token_bytes=604030656 weight_movement_bytes=220200960 total_nvlink_bytes=824231616 weight_to_token_ratio=0.364553 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=14720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.080226125e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49894 payload_rows_remote=42298 dup_rows=15642 dup_rows_remote=9768 dedup_ratio=0.238678 router_tokens_per_expert=[0, 0, 0, 16, 1566, 0, 2039, 109, 1839, 2133, 202, 0, 0, 2561, 0, 3103, 1, 1432, 0, 0, 595, 3417, 0, 0, 0, 0, 1609, 151, 1522, 108, 0, 5, 0, 0, 1, 0, 6, 2, 0, 2, 0, 0, 703, 0, 1, 0, 1984, 0, 0, 2098, 539, 0, 1449, 0, 0, 3, 1513, 2, 0, 31, 29, 299, 296, 1402] payload_rows_per_dst=[6458, 7596, 7816, 5160, 6218, 4960, 5848, 5838] dup_rows_per_dst=[1002, 5874, 3574, 1630, 10, 416, 1830, 1306] combine_rows_per_src=[6458, 7596, 7816, 5160, 6218, 4960, 5848, 5838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51174 payload_rows_remote=43644 dup_rows=14362 dup_rows_remote=11954 dedup_ratio=0.219147 router_tokens_per_expert=[0, 0, 0, 5, 0, 0, 1085, 0, 0, 109, 0, 0, 0, 0, 0, 0, 3082, 0, 0, 743, 0, 3302, 3956, 0, 0, 0, 2608, 4009, 0, 1253, 0, 0, 0, 39, 0, 0, 0, 2709, 4, 4, 2283, 0, 2576, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 3739, 0, 0, 1230, 0, 0, 0, 0] payload_rows_per_dst=[2174, 6836, 7360, 6632, 5494, 7108, 8040, 7530] dup_rows_per_dst=[6, 180, 2408, 2310, 18, 2610, 4422, 2408] combine_rows_per_src=[2174, 6836, 7360, 6632, 5494, 7108, 8040, 7530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55220 payload_rows_remote=47644 dup_rows=10316 dup_rows_remote=6286 dedup_ratio=0.157410 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 1532, 0, 0, 130, 0, 0, 0, 0, 0, 0, 3461, 0, 0, 780, 0, 2421, 3969, 0, 0, 0, 2643, 4088, 0, 850, 0, 0, 0, 80, 0, 0, 0, 2560, 2, 0, 2706, 0, 3097, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 3610, 0, 1, 770, 0, 0, 0, 0] payload_rows_per_dst=[7320, 8178, 5528, 6126, 5210, 7576, 7938, 7344] dup_rows_per_dst=[2670, 258, 874, 860, 74, 4030, 132, 1418] combine_rows_per_src=[7320, 8178, 5528, 6126, 5210, 7576, 7938, 7344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51672 payload_rows_remote=44408 dup_rows=13864 dup_rows_remote=11256 dedup_ratio=0.211548 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 1587, 0, 0, 107, 0, 0, 0, 0, 0, 0, 3348, 0, 0, 690, 0, 2422, 3434, 0, 0, 0, 3110, 4036, 0, 1065, 0, 0, 0, 35, 0, 0, 0, 2769, 60, 1, 1898, 0, 2805, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 3745, 0, 0, 1595, 0, 0, 0, 6] payload_rows_per_dst=[7264, 8076, 7480, 7242, 7752, 3796, 6868, 3194] dup_rows_per_dst=[2608, 210, 4554, 2788, 3588, 0, 108, 8] combine_rows_per_src=[7264, 8076, 7480, 7242, 7752, 3796, 6868, 3194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54542 payload_rows_remote=49364 dup_rows=10994 dup_rows_remote=10948 dedup_ratio=0.167755 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 1933, 0, 0, 122, 0, 0, 0, 0, 0, 0, 2993, 0, 0, 893, 0, 2418, 3573, 0, 0, 0, 2749, 4063, 0, 1497, 0, 0, 0, 65, 0, 0, 0, 2542, 4, 1, 2557, 0, 2773, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 3669, 1, 0, 867, 0, 0, 0, 1] payload_rows_per_dst=[7056, 8126, 5642, 6772, 5178, 7202, 7146, 7420] dup_rows_per_dst=[2798, 244, 980, 1720, 46, 3458, 92, 1656] combine_rows_per_src=[7056, 8126, 5642, 6772, 5178, 7202, 7146, 7420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=303192064 dispatch_meta_bytes=208264 combine_payload_bytes=303192064 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=303400328 ingress_bytes=743593984 link_bytes=743593984 token_bytes=606592392 weight_movement_bytes=440401920 total_nvlink_bytes=1046994312 weight_to_token_ratio=0.726026 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=12160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.293232238e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=312840192 dispatch_meta_bytes=222392 combine_payload_bytes=312840192 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=313062584 ingress_bytes=643141632 link_bytes=643141632 token_bytes=625902776 weight_movement_bytes=330301440 total_nvlink_bytes=956204216 weight_to_token_ratio=0.527720 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=24064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.118529077e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=341512192 dispatch_meta_bytes=215720 combine_payload_bytes=341512192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=341727912 ingress_bytes=561713152 link_bytes=561713152 token_bytes=683240104 weight_movement_bytes=220200960 total_nvlink_bytes=903441064 weight_to_token_ratio=0.322289 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=18176 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.769115573e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53688 payload_rows_remote=47872 dup_rows=11848 dup_rows_remote=11198 dedup_ratio=0.180786 router_tokens_per_expert=[0, 0, 0, 1, 2, 0, 793, 0, 0, 147, 0, 0, 0, 1, 0, 0, 2741, 1, 0, 591, 0, 2641, 3863, 0, 0, 0, 2750, 4040, 0, 2322, 1, 0, 2, 105, 0, 0, 0, 3035, 8, 4, 1758, 0, 2940, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 3658, 0, 1, 1340, 0, 0, 0, 6] payload_rows_per_dst=[5946, 8080, 5816, 7222, 7954, 3516, 7726, 7428] dup_rows_per_dst=[1128, 296, 650, 2924, 4234, 0, 34, 2582] combine_rows_per_src=[5946, 8080, 5816, 7222, 7954, 3516, 7726, 7428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54756 payload_rows_remote=48524 dup_rows=10780 dup_rows_remote=9272 dedup_ratio=0.164490 router_tokens_per_expert=[0, 0, 0, 5, 0, 0, 1196, 0, 0, 131, 0, 0, 0, 0, 0, 0, 3243, 0, 0, 787, 0, 2658, 3736, 0, 0, 0, 2328, 4012, 0, 1542, 0, 0, 0, 191, 0, 0, 0, 3112, 1, 2, 1947, 0, 2896, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 3704, 0, 49, 1152, 0, 0, 0, 16] payload_rows_per_dst=[6884, 8046, 5908, 6232, 6654, 6018, 7472, 7542] dup_rows_per_dst=[2004, 240, 982, 1508, 1610, 2016, 120, 2300] combine_rows_per_src=[6884, 8046, 5908, 6232, 6654, 6018, 7472, 7542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=318316544 dispatch_meta_bytes=222656 combine_payload_bytes=318316544 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=318539200 ingress_bytes=428417024 link_bytes=428417024 token_bytes=636855744 weight_movement_bytes=110100480 total_nvlink_bytes=746956224 weight_to_token_ratio=0.172881 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.450876672e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54464 payload_rows_remote=46792 dup_rows=11072 dup_rows_remote=10946 dedup_ratio=0.168945 router_tokens_per_expert=[0, 0, 0, 12, 0, 0, 1561, 0, 0, 124, 0, 0, 0, 0, 0, 0, 3233, 0, 0, 774, 0, 3252, 3836, 0, 0, 0, 2205, 4071, 0, 1117, 0, 0, 0, 121, 0, 0, 0, 2807, 6, 0, 2645, 0, 2382, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 3695, 0, 12, 852, 0, 0, 0, 0] payload_rows_per_dst=[5310, 8142, 7332, 5752, 5698, 7088, 7672, 7470] dup_rows_per_dst=[1504, 248, 3518, 892, 170, 2966, 126, 1648] combine_rows_per_src=[5310, 8142, 7332, 5752, 5698, 7088, 7672, 7470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=353841152 dispatch_meta_bytes=241248 combine_payload_bytes=353841152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=354082400 ingress_bytes=463941632 link_bytes=463941632 token_bytes=707923552 weight_movement_bytes=110100480 total_nvlink_bytes=818024032 weight_to_token_ratio=0.155526 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.068708033e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=343146496 dispatch_meta_bytes=236280 combine_payload_bytes=343146496 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=343382776 ingress_bytes=783548416 link_bytes=783548416 token_bytes=686529272 weight_movement_bytes=440401920 total_nvlink_bytes=1126931192 weight_to_token_ratio=0.641490 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.362719567e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=347820032 dispatch_meta_bytes=231184 combine_payload_bytes=347820032 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=348051216 ingress_bytes=568020992 link_bytes=568020992 token_bytes=695871248 weight_movement_bytes=220200960 total_nvlink_bytes=916072208 weight_to_token_ratio=0.316439 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.878819285e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=335405056 dispatch_meta_bytes=230952 combine_payload_bytes=335405056 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=335636008 ingress_bytes=555606016 link_bytes=555606016 token_bytes=671041064 weight_movement_bytes=220200960 total_nvlink_bytes=891242024 weight_to_token_ratio=0.328148 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=30208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.662902433e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=41930 payload_rows_remote=33740 dup_rows=23606 dup_rows_remote=10954 dedup_ratio=0.360199 router_tokens_per_expert=[0, 0, 2780, 0, 0, 413, 0, 0, 4, 0, 0, 1446, 4, 0, 0, 0, 0, 0, 712, 0, 0, 0, 1768, 0, 0, 2155, 0, 1890, 0, 1, 2, 0, 0, 0, 0, 0, 3989, 2661, 0, 187, 1, 0, 0, 0, 0, 0, 299, 0, 0, 0, 2647, 0, 0, 0, 0, 0, 3722, 0, 0, 0, 0, 3996, 4091, 0] payload_rows_per_dst=[5972, 2900, 4534, 2410, 8172, 4458, 5294, 8190] dup_rows_per_dst=[414, 8, 426, 6, 9812, 288, 0, 12652] combine_rows_per_src=[5972, 2900, 4534, 2410, 8172, 4458, 5294, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48650 payload_rows_remote=40458 dup_rows=16886 dup_rows_remote=16202 dedup_ratio=0.257660 router_tokens_per_expert=[0, 0, 1936, 0, 0, 324, 0, 0, 0, 0, 0, 2019, 0, 0, 0, 0, 0, 0, 793, 0, 0, 0, 1793, 0, 0, 2329, 0, 2259, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 2748, 0, 424, 3, 0, 0, 0, 0, 0, 339, 0, 0, 0, 2168, 0, 0, 0, 0, 0, 3772, 0, 0, 0, 0, 3859, 4096, 0] payload_rows_per_dst=[4374, 4038, 4702, 6822, 8122, 8192, 4336, 8064] dup_rows_per_dst=[146, 0, 470, 2354, 6034, 684, 0, 7198] combine_rows_per_src=[4374, 4038, 4702, 6822, 8122, 8192, 4336, 8064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54202 payload_rows_remote=46106 dup_rows=11334 dup_rows_remote=11054 dedup_ratio=0.172943 router_tokens_per_expert=[0, 0, 0, 63, 5, 0, 1707, 0, 0, 142, 0, 0, 0, 0, 0, 0, 3657, 0, 0, 793, 0, 2459, 3815, 0, 0, 0, 2194, 4046, 0, 1022, 1, 0, 0, 154, 0, 0, 0, 2693, 1, 5, 2380, 1, 3075, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 3685, 0, 83, 777, 0, 1, 0, 4] payload_rows_per_dst=[7848, 8096, 5680, 5806, 7886, 6352, 7638, 4896] dup_rows_per_dst=[3016, 280, 824, 750, 3970, 1800, 2, 692] combine_rows_per_src=[7848, 8096, 5680, 5806, 7886, 6352, 7638, 4896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47370 payload_rows_remote=39828 dup_rows=18166 dup_rows_remote=15008 dedup_ratio=0.277191 router_tokens_per_expert=[1, 0, 2102, 0, 0, 36, 0, 0, 4, 0, 0, 1869, 2, 0, 0, 0, 0, 0, 708, 0, 0, 0, 1745, 0, 0, 2291, 0, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 3211, 0, 406, 14, 0, 0, 0, 0, 0, 312, 0, 0, 0, 2491, 0, 0, 0, 0, 0, 3724, 0, 0, 0, 0, 3888, 4091, 0] payload_rows_per_dst=[7542, 7956, 8088, 6644, 812, 8190, 8138, 0] dup_rows_per_dst=[3158, 3570, 4568, 1934, 0, 644, 4292, 0] combine_rows_per_src=[7542, 7956, 8088, 6644, 812, 8190, 8138, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53076 payload_rows_remote=45136 dup_rows=12460 dup_rows_remote=7210 dedup_ratio=0.190125 router_tokens_per_expert=[0, 0, 2986, 0, 0, 511, 0, 0, 6, 0, 0, 1430, 0, 0, 0, 0, 0, 0, 388, 0, 0, 0, 1300, 0, 0, 2437, 0, 2367, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 3130, 0, 107, 3, 0, 0, 1, 0, 0, 229, 0, 0, 0, 2536, 1, 0, 0, 0, 0, 3373, 0, 0, 0, 0, 3867, 4095, 0] payload_rows_per_dst=[6446, 7276, 4094, 7144, 7940, 8192, 5074, 6910] dup_rows_per_dst=[548, 2190, 568, 2464, 5250, 464, 0, 976] combine_rows_per_src=[6446, 7276, 4094, 7144, 7940, 8192, 5074, 6910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=241848320 dispatch_meta_bytes=178776 combine_payload_bytes=241848320 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=242027096 ingress_bytes=682250240 link_bytes=682250240 token_bytes=483875416 weight_movement_bytes=440401920 total_nvlink_bytes=924277336 weight_to_token_ratio=0.910156 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=17792 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.186545378e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=290002944 dispatch_meta_bytes=226640 combine_payload_bytes=290002944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=290229584 ingress_bytes=400103424 link_bytes=400103424 token_bytes=580232528 weight_movement_bytes=110100480 total_nvlink_bytes=690333008 weight_to_token_ratio=0.189752 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.958456600e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47176 payload_rows_remote=39052 dup_rows=18360 dup_rows_remote=15906 dedup_ratio=0.280151 router_tokens_per_expert=[0, 0, 3307, 0, 0, 63, 0, 0, 3, 0, 0, 1541, 6, 0, 0, 0, 0, 0, 352, 0, 0, 0, 922, 0, 0, 1893, 0, 2380, 0, 0, 4, 0, 0, 0, 0, 0, 4015, 2819, 0, 138, 15, 0, 0, 0, 0, 0, 217, 0, 0, 0, 3345, 1, 0, 0, 0, 0, 3770, 0, 0, 0, 0, 3890, 4087, 0] payload_rows_per_dst=[7742, 7992, 8124, 6680, 276, 8190, 8172, 0] dup_rows_per_dst=[4636, 2888, 2454, 1874, 0, 448, 6060, 0] combine_rows_per_src=[7742, 7992, 8124, 6680, 276, 8190, 8172, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=330487808 dispatch_meta_bytes=228640 combine_payload_bytes=330487808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=330716448 ingress_bytes=440588288 link_bytes=440588288 token_bytes=661204256 weight_movement_bytes=110100480 total_nvlink_bytes=771304736 weight_to_token_ratio=0.166515 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.662554972e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=285487104 dispatch_meta_bytes=219344 combine_payload_bytes=285487104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=285706448 ingress_bytes=395587584 link_bytes=395587584 token_bytes=571193552 weight_movement_bytes=110100480 total_nvlink_bytes=681294032 weight_to_token_ratio=0.192755 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.879918716e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50306 payload_rows_remote=45306 dup_rows=15230 dup_rows_remote=15230 dedup_ratio=0.232391 router_tokens_per_expert=[2, 0, 2279, 0, 0, 381, 0, 0, 0, 0, 0, 2319, 0, 0, 0, 0, 0, 0, 649, 0, 0, 0, 1889, 0, 0, 2547, 0, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 2575, 0, 217, 2, 0, 0, 0, 0, 0, 214, 0, 0, 0, 2500, 0, 0, 0, 0, 0, 3587, 0, 0, 0, 0, 3853, 4096, 0] payload_rows_per_dst=[5068, 4638, 4618, 6592, 8098, 8192, 5000, 8100] dup_rows_per_dst=[256, 0, 458, 1742, 5562, 432, 0, 6780] combine_rows_per_src=[5068, 4638, 4618, 6592, 8098, 8192, 5000, 8100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54020 payload_rows_remote=46944 dup_rows=11516 dup_rows_remote=8790 dedup_ratio=0.175720 router_tokens_per_expert=[0, 0, 2402, 0, 0, 188, 0, 0, 6, 0, 0, 1896, 4, 0, 0, 0, 0, 0, 455, 0, 0, 0, 1222, 0, 0, 2522, 0, 2377, 0, 0, 2, 0, 0, 0, 0, 0, 4018, 3328, 0, 91, 7, 0, 0, 0, 0, 0, 162, 0, 0, 0, 2608, 0, 0, 0, 0, 0, 3641, 0, 0, 0, 0, 3748, 4091, 0] payload_rows_per_dst=[5590, 7892, 8148, 7076, 5162, 8190, 5296, 6666] dup_rows_per_dst=[1182, 3416, 3242, 2726, 84, 330, 536, 0] combine_rows_per_src=[5590, 7892, 8148, 7076, 5162, 8190, 5296, 6666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=323534848 dispatch_meta_bytes=209384 combine_payload_bytes=323534848 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=323744232 ingress_bytes=653836288 link_bytes=653836288 token_bytes=647279080 weight_movement_bytes=330301440 total_nvlink_bytes=977580520 weight_to_token_ratio=0.510292 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=15232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.137128842e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279924736 dispatch_meta_bytes=219832 combine_payload_bytes=279924736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280144568 ingress_bytes=390025216 link_bytes=390025216 token_bytes=560069304 weight_movement_bytes=110100480 total_nvlink_bytes=670169784 weight_to_token_ratio=0.196584 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.783179987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=324753408 dispatch_meta_bytes=242144 combine_payload_bytes=324753408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=324995552 ingress_bytes=434853888 link_bytes=434853888 token_bytes=649748960 weight_movement_bytes=110100480 total_nvlink_bytes=759849440 weight_to_token_ratio=0.169451 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.562824324e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54812 payload_rows_remote=46620 dup_rows=10724 dup_rows_remote=7976 dedup_ratio=0.163635 router_tokens_per_expert=[0, 19, 0, 0, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 17, 2395, 0, 4, 0, 0, 4080, 3988, 0, 4087, 0, 0, 0, 0, 0, 4084, 166, 0, 0, 2421, 0, 0, 0, 810, 0, 0, 0, 745, 4021, 0, 0, 0, 371, 0, 0, 15, 1000, 0] payload_rows_per_dst=[982, 8174, 8146, 4816, 8160, 8176, 8166, 8192] dup_rows_per_dst=[0, 0, 0, 16, 0, 4974, 2986, 2748] combine_rows_per_src=[982, 8174, 8146, 4816, 8160, 8176, 8166, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=336494592 dispatch_meta_bytes=222936 combine_payload_bytes=336494592 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=336717528 ingress_bytes=556695552 link_bytes=556695552 token_bytes=673212120 weight_movement_bytes=220200960 total_nvlink_bytes=893413080 weight_to_token_ratio=0.327090 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.681851256e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59636 payload_rows_remote=54160 dup_rows=5900 dup_rows_remote=5814 dedup_ratio=0.090027 router_tokens_per_expert=[0, 1, 0, 0, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 30, 2590, 0, 1, 0, 0, 4094, 4016, 0, 4092, 0, 0, 0, 0, 0, 4091, 164, 0, 0, 2616, 1, 0, 0, 710, 0, 0, 0, 462, 4003, 0, 0, 0, 158, 0, 0, 40, 623, 0] payload_rows_per_dst=[8190, 8184, 8156, 5226, 8032, 5476, 8182, 8190] dup_rows_per_dst=[1996, 0, 0, 16, 0, 86, 2168, 1634] combine_rows_per_src=[8190, 8184, 8156, 5226, 8032, 5476, 8182, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47182 payload_rows_remote=39298 dup_rows=18354 dup_rows_remote=15466 dedup_ratio=0.280060 router_tokens_per_expert=[0, 0, 2907, 0, 0, 322, 0, 0, 0, 0, 0, 1609, 5, 0, 0, 0, 0, 0, 573, 0, 0, 0, 953, 0, 0, 2087, 0, 2442, 0, 2, 3, 0, 0, 0, 0, 0, 3995, 3002, 0, 106, 26, 0, 0, 0, 0, 0, 216, 0, 0, 0, 3061, 0, 0, 0, 0, 0, 3608, 0, 0, 0, 0, 3772, 4079, 0] payload_rows_per_dst=[7778, 7884, 8128, 6864, 212, 8182, 8134, 0] dup_rows_per_dst=[4684, 2888, 2914, 2204, 0, 460, 5204, 0] combine_rows_per_src=[7778, 7884, 8128, 6864, 212, 8182, 8134, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53394 payload_rows_remote=45202 dup_rows=12142 dup_rows_remote=10960 dedup_ratio=0.185272 router_tokens_per_expert=[0, 14, 0, 0, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 22, 2627, 0, 0, 0, 0, 4090, 4007, 0, 4069, 0, 0, 0, 0, 0, 4045, 180, 0, 0, 2415, 0, 0, 0, 444, 0, 0, 0, 1172, 4012, 0, 0, 0, 153, 0, 0, 54, 805, 0] payload_rows_per_dst=[8192, 8176, 8152, 7806, 7806, 8192, 3076, 1994] dup_rows_per_dst=[1182, 170, 804, 4712, 0, 5088, 156, 30] combine_rows_per_src=[8192, 8176, 8152, 7806, 7806, 8192, 3076, 1994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58406 payload_rows_remote=50252 dup_rows=7130 dup_rows_remote=7130 dedup_ratio=0.108795 router_tokens_per_expert=[0, 5, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 110, 2340, 0, 2, 0, 0, 4089, 4077, 0, 4070, 0, 0, 0, 0, 0, 4069, 349, 0, 0, 2133, 0, 0, 0, 964, 0, 0, 0, 837, 3939, 0, 0, 0, 716, 0, 0, 28, 477, 0] payload_rows_per_dst=[8186, 8140, 8082, 4820, 8154, 4704, 8136, 8184] dup_rows_per_dst=[1046, 0, 0, 84, 0, 260, 3344, 2396] combine_rows_per_src=[8186, 8140, 8082, 4820, 8154, 4704, 8136, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=334172160 dispatch_meta_bytes=218384 combine_payload_bytes=334172160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=334390544 ingress_bytes=444272640 link_bytes=444272640 token_bytes=668562704 weight_movement_bytes=110100480 total_nvlink_bytes=778663184 weight_to_token_ratio=0.164682 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.726631913e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=388218880 dispatch_meta_bytes=239896 combine_payload_bytes=388218880 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=388458776 ingress_bytes=608419840 link_bytes=608419840 token_bytes=776677656 weight_movement_bytes=220200960 total_nvlink_bytes=996878616 weight_to_token_ratio=0.283517 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058142170e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55648 payload_rows_remote=47640 dup_rows=9888 dup_rows_remote=9888 dedup_ratio=0.150879 router_tokens_per_expert=[0, 13, 0, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 21, 2820, 0, 9, 0, 0, 4084, 3827, 0, 4083, 0, 0, 0, 0, 0, 4057, 405, 0, 0, 1944, 0, 0, 0, 789, 0, 0, 0, 1142, 3901, 0, 0, 0, 834, 0, 0, 1, 418, 0] payload_rows_per_dst=[8190, 8166, 8008, 7742, 7722, 5186, 3548, 7086] dup_rows_per_dst=[836, 0, 0, 4708, 984, 1340, 314, 1706] combine_rows_per_src=[8190, 8166, 8008, 7742, 7722, 5186, 3548, 7086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=281688064 dispatch_meta_bytes=219056 combine_payload_bytes=281688064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281907120 ingress_bytes=391788544 link_bytes=391788544 token_bytes=563595184 weight_movement_bytes=110100480 total_nvlink_bytes=673695664 weight_to_token_ratio=0.195354 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.813847162e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=324007936 dispatch_meta_bytes=224648 combine_payload_bytes=324007936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=324232584 ingress_bytes=434108416 link_bytes=434108416 token_bytes=648240520 weight_movement_bytes=110100480 total_nvlink_bytes=758341000 weight_to_token_ratio=0.169845 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=28160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.549859340e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=360206336 dispatch_meta_bytes=229528 combine_payload_bytes=360206336 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=360435864 ingress_bytes=910708736 link_bytes=910708736 token_bytes=720642200 weight_movement_bytes=550502400 total_nvlink_bytes=1271144600 weight_to_token_ratio=0.763905 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=26880 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.583872278e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57496 payload_rows_remote=49364 dup_rows=8040 dup_rows_remote=4646 dedup_ratio=0.122681 router_tokens_per_expert=[0, 0, 0, 0, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 23, 2684, 0, 0, 0, 0, 4092, 4023, 0, 4096, 0, 0, 0, 0, 0, 4073, 244, 0, 0, 1759, 0, 0, 0, 527, 0, 0, 0, 1257, 3979, 0, 0, 0, 286, 0, 0, 59, 1066, 0] payload_rows_per_dst=[7362, 8192, 8180, 5402, 7170, 4868, 8132, 8190] dup_rows_per_dst=[848, 0, 0, 12, 0, 1008, 3394, 2778] combine_rows_per_src=[7362, 8192, 8180, 5402, 7170, 4868, 8132, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57830 payload_rows_remote=51346 dup_rows=7706 dup_rows_remote=7664 dedup_ratio=0.117584 router_tokens_per_expert=[0, 11, 0, 0, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 25, 3233, 0, 5, 0, 0, 4083, 3934, 0, 4089, 0, 0, 0, 0, 0, 4042, 296, 0, 0, 1536, 0, 0, 0, 754, 0, 0, 0, 1149, 4007, 0, 0, 0, 355, 0, 0, 1, 772, 0] payload_rows_per_dst=[8184, 8178, 8046, 6484, 7940, 3462, 7370, 8166] dup_rows_per_dst=[910, 0, 0, 42, 1646, 202, 2732, 2174] combine_rows_per_src=[8184, 8178, 8046, 6484, 7940, 3462, 7370, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=341483520 dispatch_meta_bytes=230112 combine_payload_bytes=341483520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=341713632 ingress_bytes=451584000 link_bytes=451584000 token_bytes=683197152 weight_movement_bytes=110100480 total_nvlink_bytes=793297632 weight_to_token_ratio=0.161155 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.853788488e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=353841152 dispatch_meta_bytes=216040 combine_payload_bytes=353841152 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=354057192 ingress_bytes=684142592 link_bytes=684142592 token_bytes=707898344 weight_movement_bytes=330301440 total_nvlink_bytes=1038199784 weight_to_token_ratio=0.466594 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.189836490e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=368048128 dispatch_meta_bytes=236040 combine_payload_bytes=368048128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368284168 ingress_bytes=478148608 link_bytes=478148608 token_bytes=736332296 weight_movement_bytes=110100480 total_nvlink_bytes=846432776 weight_to_token_ratio=0.149526 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=21632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.315790712e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59622 payload_rows_remote=51430 dup_rows=5914 dup_rows_remote=2454 dedup_ratio=0.090240 router_tokens_per_expert=[0, 3805, 0, 0, 0, 10, 2989, 4095, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 2903, 0, 0, 4096, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1689, 0, 40, 1, 0, 0, 4096, 0, 0, 7, 123, 0, 0, 38, 0, 0, 0, 3163, 1512, 0, 0, 0] payload_rows_per_dst=[5998, 8140, 5806, 7610, 8190, 8192, 8192, 7494] dup_rows_per_dst=[0, 0, 0, 262, 0, 3460, 336, 1856] combine_rows_per_src=[5998, 8140, 5806, 7610, 8190, 8192, 8192, 7494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52874 payload_rows_remote=45096 dup_rows=12662 dup_rows_remote=8536 dedup_ratio=0.193207 router_tokens_per_expert=[0, 3793, 0, 0, 0, 31, 3041, 4096, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 3, 2045, 0, 0, 4096, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 0, 64, 0, 0, 0, 4096, 0, 0, 20, 98, 0, 0, 31, 0, 0, 0, 2985, 2009, 0, 0, 0] payload_rows_per_dst=[2280, 8116, 4092, 7876, 8192, 8192, 6348, 7778] dup_rows_per_dst=[30, 0, 4, 3780, 6, 4490, 226, 4126] combine_rows_per_src=[2280, 8116, 4092, 7876, 8192, 8192, 6348, 7778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58288 payload_rows_remote=50098 dup_rows=7248 dup_rows_remote=7248 dedup_ratio=0.110596 router_tokens_per_expert=[0, 3547, 0, 0, 0, 25, 3105, 4095, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 1, 4, 2281, 0, 0, 4095, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2589, 0, 123, 0, 0, 0, 4096, 0, 0, 23, 114, 0, 0, 34, 0, 0, 0, 3344, 1119, 0, 0, 0] payload_rows_per_dst=[6236, 8068, 5936, 7132, 8190, 7012, 8192, 7522] dup_rows_per_dst=[24, 0, 2254, 240, 0, 2984, 342, 1404] combine_rows_per_src=[6236, 8068, 5936, 7132, 8190, 7012, 8192, 7522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53914 payload_rows_remote=46080 dup_rows=11622 dup_rows_remote=7830 dedup_ratio=0.177338 router_tokens_per_expert=[0, 3669, 0, 0, 0, 17, 3236, 4094, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 6, 2560, 0, 0, 4096, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1971, 0, 77, 0, 0, 0, 4096, 0, 0, 18, 101, 0, 0, 32, 0, 0, 0, 3450, 1152, 0, 0, 0] payload_rows_per_dst=[7834, 8164, 8192, 7360, 8188, 4086, 7786, 2304] dup_rows_per_dst=[3792, 410, 6912, 198, 2, 10, 298, 0] combine_rows_per_src=[7834, 8164, 8192, 7360, 8188, 4086, 7786, 2304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53948 payload_rows_remote=45792 dup_rows=11588 dup_rows_remote=11588 dedup_ratio=0.176819 router_tokens_per_expert=[0, 16, 1, 0, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 21, 2870, 0, 1, 0, 0, 4022, 3961, 0, 4078, 0, 0, 0, 0, 4, 3994, 264, 0, 0, 1995, 0, 0, 0, 566, 0, 0, 0, 1350, 3933, 0, 0, 0, 292, 0, 0, 15, 727, 0] payload_rows_per_dst=[8066, 8156, 7960, 8160, 7922, 8184, 3482, 2018] dup_rows_per_dst=[1366, 0, 2, 5490, 0, 4330, 350, 50] combine_rows_per_src=[8066, 8156, 7960, 8160, 7922, 8184, 3482, 2018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=368650240 dispatch_meta_bytes=215536 combine_payload_bytes=368650240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368865776 ingress_bytes=478750720 link_bytes=478750720 token_bytes=737516016 weight_movement_bytes=110100480 total_nvlink_bytes=847616496 weight_to_token_ratio=0.149286 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.326262430e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60162 payload_rows_remote=51972 dup_rows=5374 dup_rows_remote=5366 dedup_ratio=0.082001 router_tokens_per_expert=[0, 3764, 0, 0, 0, 144, 3310, 4095, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 4, 2154, 0, 0, 4095, 0, 0, 0, 0, 140, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2176, 0, 97, 2, 0, 0, 4096, 0, 0, 19, 14, 0, 0, 39, 0, 0, 0, 3308, 1240, 0, 0, 0] payload_rows_per_dst=[7892, 8138, 8190, 7538, 8190, 4526, 8192, 7496] dup_rows_per_dst=[3324, 0, 8, 272, 2, 24, 144, 1600] combine_rows_per_src=[7892, 8138, 8190, 7538, 8190, 4526, 8192, 7496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=330301440 dispatch_meta_bytes=215640 combine_payload_bytes=330301440 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=330517080 ingress_bytes=880803840 link_bytes=880803840 token_bytes=660818520 weight_movement_bytes=550502400 total_nvlink_bytes=1211320920 weight_to_token_ratio=0.833061 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.531862745e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=323248128 dispatch_meta_bytes=214528 combine_payload_bytes=323248128 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=323462656 ingress_bytes=543449088 link_bytes=543449088 token_bytes=646710784 weight_movement_bytes=220200960 total_nvlink_bytes=866911744 weight_to_token_ratio=0.340494 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.451473460e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=359102464 dispatch_meta_bytes=229384 combine_payload_bytes=359102464 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=359331848 ingress_bytes=579303424 link_bytes=579303424 token_bytes=718434312 weight_movement_bytes=220200960 total_nvlink_bytes=938635272 weight_to_token_ratio=0.306501 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.007503933e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=328237056 dispatch_meta_bytes=229520 combine_payload_bytes=328237056 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=328466576 ingress_bytes=548438016 link_bytes=548438016 token_bytes=656703632 weight_movement_bytes=220200960 total_nvlink_bytes=876904592 weight_to_token_ratio=0.335313 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.538239124e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58538 payload_rows_remote=50346 dup_rows=6998 dup_rows_remote=6692 dedup_ratio=0.106781 router_tokens_per_expert=[0, 3860, 0, 0, 0, 21, 2698, 4096, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 2652, 0, 0, 4096, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2089, 0, 44, 0, 0, 0, 4096, 0, 0, 18, 100, 0, 0, 35, 0, 0, 0, 2999, 1713, 0, 0, 0] payload_rows_per_dst=[5410, 8158, 5304, 7740, 8192, 8192, 8192, 7350] dup_rows_per_dst=[28, 0, 0, 324, 0, 4266, 306, 2074] combine_rows_per_src=[5410, 8158, 5304, 7740, 8192, 8192, 8192, 7350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59746 payload_rows_remote=52434 dup_rows=5790 dup_rows_remote=5582 dedup_ratio=0.088348 router_tokens_per_expert=[0, 3638, 0, 0, 0, 36, 3134, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 3, 2490, 0, 0, 4096, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2254, 0, 102, 1, 0, 0, 4096, 0, 0, 4, 111, 0, 0, 33, 0, 0, 0, 3123, 1347, 0, 0, 0] payload_rows_per_dst=[7682, 8164, 8192, 7312, 8192, 4684, 8192, 7328] dup_rows_per_dst=[3122, 0, 522, 208, 0, 30, 296, 1612] combine_rows_per_src=[7682, 8164, 8192, 7312, 8192, 4684, 8192, 7328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=372535296 dispatch_meta_bytes=229352 combine_payload_bytes=372535296 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=372764648 ingress_bytes=702836736 link_bytes=702836736 token_bytes=745299944 weight_movement_bytes=330301440 total_nvlink_bytes=1075601384 weight_to_token_ratio=0.443179 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=18176 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.222348681e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=360880128 dispatch_meta_bytes=228152 combine_payload_bytes=360880128 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=361108280 ingress_bytes=581081088 link_bytes=581081088 token_bytes=721988408 weight_movement_bytes=220200960 total_nvlink_bytes=942189368 weight_to_token_ratio=0.304992 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.010595583e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=375846912 dispatch_meta_bytes=232064 combine_payload_bytes=375846912 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=376078976 ingress_bytes=596047872 link_bytes=596047872 token_bytes=751925888 weight_movement_bytes=220200960 total_nvlink_bytes=972126848 weight_to_token_ratio=0.292849 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.036625282e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61524 payload_rows_remote=53332 dup_rows=4012 dup_rows_remote=4012 dedup_ratio=0.061218 router_tokens_per_expert=[0, 4086, 0, 0, 0, 2409, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 4093, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 83, 453, 4095, 0, 0, 0, 4096, 564, 0, 6, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4818, 8186, 7680, 8092, 8172, 8192, 8192, 8192] dup_rows_per_dst=[0, 152, 0, 0, 1650, 0, 2198, 12] combine_rows_per_src=[4818, 8186, 7680, 8092, 8172, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58472 payload_rows_remote=50280 dup_rows=7064 dup_rows_remote=7022 dedup_ratio=0.107788 router_tokens_per_expert=[0, 4088, 0, 0, 0, 1340, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 0, 4090, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 39, 564, 4094, 0, 0, 0, 4096, 1344, 0, 21, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2680, 7324, 7582, 8134, 8176, 8192, 8192, 8192] dup_rows_per_dst=[0, 22, 0, 720, 2250, 8, 4022, 42] combine_rows_per_src=[2680, 7324, 7582, 8134, 8176, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60954 payload_rows_remote=52804 dup_rows=4582 dup_rows_remote=3102 dedup_ratio=0.069916 router_tokens_per_expert=[0, 4075, 0, 0, 0, 2130, 0, 0, 63, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 4078, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 115, 521, 4094, 0, 0, 0, 4096, 829, 0, 25, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4260, 8158, 7702, 8108, 8150, 8192, 8192, 8192] dup_rows_per_dst=[0, 126, 0, 0, 1480, 0, 2926, 50] combine_rows_per_src=[4260, 8158, 7702, 8108, 8150, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52368 payload_rows_remote=44176 dup_rows=13168 dup_rows_remote=384 dedup_ratio=0.200928 router_tokens_per_expert=[0, 4053, 0, 0, 0, 2351, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 0, 4088, 0, 0, 0, 0, 3986, 0, 0, 0, 0, 1316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 38, 364, 4084, 0, 0, 0, 4096, 261, 0, 47, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8176, 7778, 7972, 2632, 8192, 1558, 7868] dup_rows_per_dst=[12784, 198, 0, 0, 0, 0, 186, 0] combine_rows_per_src=[8192, 8176, 7778, 7972, 2632, 8192, 1558, 7868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58500 payload_rows_remote=50322 dup_rows=7036 dup_rows_remote=7036 dedup_ratio=0.107361 router_tokens_per_expert=[0, 3704, 0, 0, 0, 34, 3004, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4, 2904, 0, 0, 4076, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2538, 0, 154, 2, 0, 0, 4096, 0, 0, 2, 119, 0, 0, 30, 0, 0, 0, 2700, 1078, 0, 0, 0] payload_rows_per_dst=[7978, 8178, 8190, 7462, 8192, 5310, 8192, 4998] dup_rows_per_dst=[3906, 0, 1934, 222, 0, 78, 302, 594] combine_rows_per_src=[7978, 8178, 8190, 7462, 8192, 5310, 8192, 4998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=382283776 dispatch_meta_bytes=229376 combine_payload_bytes=382283776 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382513152 ingress_bytes=492384256 link_bytes=492384256 token_bytes=764796928 weight_movement_bytes=110100480 total_nvlink_bytes=874897408 weight_to_token_ratio=0.143960 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.563372045e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58678 payload_rows_remote=50766 dup_rows=6858 dup_rows_remote=2912 dedup_ratio=0.104645 router_tokens_per_expert=[0, 4074, 0, 0, 0, 2356, 0, 1, 22, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 0, 4095, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 103, 360, 4085, 0, 0, 0, 4096, 961, 0, 25, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5570, 8190, 7912, 8168, 8150, 8190, 4306, 8192] dup_rows_per_dst=[934, 46, 3946, 0, 1226, 0, 656, 50] combine_rows_per_src=[5570, 8190, 7912, 8168, 8150, 8190, 4306, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=360407040 dispatch_meta_bytes=229208 combine_payload_bytes=360407040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=360636248 ingress_bytes=470507520 link_bytes=470507520 token_bytes=721043288 weight_movement_bytes=110100480 total_nvlink_bytes=831143768 weight_to_token_ratio=0.152696 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.182899625e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=378499072 dispatch_meta_bytes=223624 combine_payload_bytes=378499072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=378722696 ingress_bytes=488599552 link_bytes=488599552 token_bytes=757221768 weight_movement_bytes=110100480 total_nvlink_bytes=867322248 weight_to_token_ratio=0.145401 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.497549817e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=316653568 dispatch_meta_bytes=178240 combine_payload_bytes=316653568 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=316831808 ingress_bytes=536854528 link_bytes=536854528 token_bytes=633485376 weight_movement_bytes=220200960 total_nvlink_bytes=853686336 weight_to_token_ratio=0.347602 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.336783216e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61006 payload_rows_remote=52814 dup_rows=4530 dup_rows_remote=1334 dedup_ratio=0.069122 router_tokens_per_expert=[0, 4080, 0, 0, 0, 2199, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 0, 4096, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 46, 944, 4096, 0, 0, 0, 4096, 608, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4398, 8192, 7606, 8074, 8160, 8192, 8192, 8192] dup_rows_per_dst=[0, 86, 0, 0, 1246, 0, 3196, 2] combine_rows_per_src=[4398, 8192, 7606, 8074, 8160, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=360708096 dispatch_meta_bytes=229432 combine_payload_bytes=360708096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=360937528 ingress_bytes=470808576 link_bytes=470808576 token_bytes=721645624 weight_movement_bytes=110100480 total_nvlink_bytes=831746104 weight_to_token_ratio=0.152569 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.188135484e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60934 payload_rows_remote=52822 dup_rows=4602 dup_rows_remote=4602 dedup_ratio=0.070221 router_tokens_per_expert=[0, 4072, 0, 0, 0, 2091, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 0, 4089, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 65, 514, 4092, 0, 0, 0, 4096, 732, 0, 12, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4182, 8178, 7742, 8112, 8144, 8192, 8192, 8192] dup_rows_per_dst=[0, 136, 0, 0, 1828, 0, 2614, 24] combine_rows_per_src=[4182, 8178, 7742, 8112, 8144, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=363890688 dispatch_meta_bytes=214712 combine_payload_bytes=363890688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364105400 ingress_bytes=473991168 link_bytes=473991168 token_bytes=727996088 weight_movement_bytes=110100480 total_nvlink_bytes=838096568 weight_to_token_ratio=0.151238 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.243485993e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60020 payload_rows_remote=52354 dup_rows=5516 dup_rows_remote=5396 dedup_ratio=0.084167 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 17, 0, 4093, 3, 0, 0, 0, 0, 26, 2102, 0, 0, 0, 0, 4096, 0, 0, 0, 3832, 4096, 3753, 0, 3395, 0, 0, 0, 0, 3304, 0, 0, 0, 0, 0, 0, 2, 0, 0, 59, 0, 0, 0, 0, 0, 0, 3986, 0, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 4214, 7506, 8090, 7666, 7972, 8192] dup_rows_per_dst=[4, 38, 42, 0, 5308, 120, 0, 4] combine_rows_per_src=[8192, 8188, 4214, 7506, 8090, 7666, 7972, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=378570752 dispatch_meta_bytes=216592 combine_payload_bytes=378570752 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=378787344 ingress_bytes=818972672 link_bytes=818972672 token_bytes=757358096 weight_movement_bytes=440401920 total_nvlink_bytes=1197760016 weight_to_token_ratio=0.581498 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=21632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.424328174e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=378628096 dispatch_meta_bytes=229696 combine_payload_bytes=378628096 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=378857792 ingress_bytes=708929536 link_bytes=708929536 token_bytes=757485888 weight_movement_bytes=330301440 total_nvlink_bytes=1087787328 weight_to_token_ratio=0.436050 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.232945062e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59816 payload_rows_remote=52676 dup_rows=5720 dup_rows_remote=1418 dedup_ratio=0.087280 router_tokens_per_expert=[0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 27, 0, 4094, 3, 0, 0, 0, 0, 36, 2295, 0, 0, 0, 0, 4096, 0, 0, 0, 3973, 4096, 3709, 0, 3447, 0, 0, 0, 0, 3034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 2, 0, 0, 0, 0, 0, 3857, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 5022, 7418, 7140, 7946, 7714, 8192] dup_rows_per_dst=[12, 56, 1160, 0, 4302, 184, 4, 2] combine_rows_per_src=[8192, 8192, 5022, 7418, 7140, 7946, 7714, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58478 payload_rows_remote=50286 dup_rows=7058 dup_rows_remote=7058 dedup_ratio=0.107697 router_tokens_per_expert=[0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 19, 0, 4095, 0, 0, 0, 0, 0, 26, 2153, 0, 0, 0, 0, 4096, 0, 0, 0, 3829, 4096, 3936, 0, 3516, 0, 0, 0, 0, 2909, 0, 0, 0, 0, 0, 0, 3, 0, 0, 52, 3, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8084, 7922, 4326, 6274, 7764, 7796, 8120, 8192] dup_rows_per_dst=[12, 36, 32, 0, 2954, 1678, 2346, 0] combine_rows_per_src=[8084, 7922, 4326, 6274, 7764, 7796, 8120, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=375273472 dispatch_meta_bytes=231000 combine_payload_bytes=375273472 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=375504472 ingress_bytes=595474432 link_bytes=595474432 token_bytes=750777944 weight_movement_bytes=220200960 total_nvlink_bytes=970978904 weight_to_token_ratio=0.293297 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.035627976e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57308 payload_rows_remote=49116 dup_rows=8228 dup_rows_remote=8214 dedup_ratio=0.125549 router_tokens_per_expert=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 43, 0, 4083, 0, 0, 0, 0, 0, 26, 1963, 0, 0, 0, 0, 4096, 0, 0, 0, 4040, 4096, 3820, 0, 3344, 0, 0, 0, 0, 3160, 0, 0, 0, 0, 0, 0, 5, 0, 0, 78, 19, 0, 0, 0, 0, 0, 3977, 0, 0, 0, 11, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8172, 8128, 500, 8088, 7954, 8082] dup_rows_per_dst=[14, 60, 2494, 5464, 0, 158, 38, 0] combine_rows_per_src=[8192, 8192, 8172, 8128, 500, 8088, 7954, 8082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54948 payload_rows_remote=46784 dup_rows=10588 dup_rows_remote=10386 dedup_ratio=0.161560 router_tokens_per_expert=[0, 4001, 0, 0, 0, 1721, 0, 0, 98, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 0, 4081, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 40, 536, 4091, 0, 0, 0, 4096, 881, 0, 72, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8164, 7588, 8156, 3594, 8186, 2876, 8192] dup_rows_per_dst=[9526, 202, 0, 0, 678, 0, 38, 144] combine_rows_per_src=[8192, 8164, 7588, 8156, 3594, 8186, 2876, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60462 payload_rows_remote=52344 dup_rows=5074 dup_rows_remote=312 dedup_ratio=0.077423 router_tokens_per_expert=[0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 26, 0, 4083, 11, 0, 0, 0, 0, 46, 3031, 0, 0, 0, 0, 4096, 0, 0, 0, 3787, 4096, 3904, 0, 3363, 0, 0, 0, 0, 2175, 0, 0, 0, 0, 0, 0, 1, 1, 0, 108, 9, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8118, 7808, 4350, 7576, 8038, 8192] dup_rows_per_dst=[22, 52, 4762, 0, 0, 218, 18, 2] combine_rows_per_src=[8192, 8188, 8118, 7808, 4350, 7576, 8038, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=377581568 dispatch_meta_bytes=216376 combine_payload_bytes=377581568 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=377797944 ingress_bytes=817983488 link_bytes=817983488 token_bytes=755379512 weight_movement_bytes=440401920 total_nvlink_bytes=1195781432 weight_to_token_ratio=0.583021 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.422607821e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=360450048 dispatch_meta_bytes=229376 combine_payload_bytes=360450048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=360679424 ingress_bytes=470550528 link_bytes=470550528 token_bytes=721129472 weight_movement_bytes=110100480 total_nvlink_bytes=831229952 weight_to_token_ratio=0.152678 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.183647605e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=352063488 dispatch_meta_bytes=229320 combine_payload_bytes=352063488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352292808 ingress_bytes=462163968 link_bytes=462163968 token_bytes=704356296 weight_movement_bytes=110100480 total_nvlink_bytes=814456776 weight_to_token_ratio=0.156314 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.037791533e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60276 payload_rows_remote=52828 dup_rows=5260 dup_rows_remote=5260 dedup_ratio=0.080261 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4093, 3, 0, 0, 0, 0, 27, 2074, 0, 0, 0, 0, 4096, 0, 0, 0, 4028, 4096, 3992, 0, 3269, 0, 0, 0, 0, 3270, 0, 0, 0, 0, 0, 0, 24, 0, 0, 55, 0, 0, 0, 0, 0, 0, 3724, 0, 0, 0, 5, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 4158, 7984, 8058, 8056, 7448, 8192] dup_rows_per_dst=[0, 28, 44, 0, 5020, 158, 0, 10] combine_rows_per_src=[8192, 8188, 4158, 7984, 8058, 8056, 7448, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61658 payload_rows_remote=53824 dup_rows=3878 dup_rows_remote=3878 dedup_ratio=0.059174 router_tokens_per_expert=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 13, 0, 4070, 8, 0, 0, 0, 0, 45, 2780, 0, 0, 0, 0, 4096, 0, 0, 0, 3868, 4096, 3917, 0, 3055, 0, 0, 0, 0, 2809, 0, 0, 0, 0, 0, 0, 14, 0, 0, 93, 7, 0, 0, 0, 0, 0, 3881, 0, 0, 0, 9, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8160, 8164, 7834, 5618, 7736, 7762, 8192] dup_rows_per_dst=[14, 22, 3596, 0, 0, 214, 14, 18] combine_rows_per_src=[8192, 8160, 8164, 7834, 5618, 7736, 7762, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=335347712 dispatch_meta_bytes=228680 combine_payload_bytes=335347712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=335576392 ingress_bytes=445448192 link_bytes=445448192 token_bytes=670924104 weight_movement_bytes=110100480 total_nvlink_bytes=781024584 weight_to_token_ratio=0.164103 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.747076695e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62828 payload_rows_remote=54636 dup_rows=2708 dup_rows_remote=2228 dedup_ratio=0.041321 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 3, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 4093, 2, 307, 2566, 0, 0, 241, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 894, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8188, 8190, 7958, 5724, 8192, 8192, 8192] dup_rows_per_dst=[0, 410, 0, 4, 26, 480, 0, 1788] combine_rows_per_src=[8192, 8188, 8190, 7958, 5724, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=375201792 dispatch_meta_bytes=210624 combine_payload_bytes=375201792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375412416 ingress_bytes=485302272 link_bytes=485302272 token_bytes=750614208 weight_movement_bytes=110100480 total_nvlink_bytes=860714688 weight_to_token_ratio=0.146681 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.440204695e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 2, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=378671104 dispatch_meta_bytes=232352 combine_payload_bytes=378671104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=378903456 ingress_bytes=488771584 link_bytes=488771584 token_bytes=757574560 weight_movement_bytes=110100480 total_nvlink_bytes=867675040 weight_to_token_ratio=0.145333 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.500541737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=385810432 dispatch_meta_bytes=230808 combine_payload_bytes=385810432 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=386041240 ingress_bytes=936312832 link_bytes=936312832 token_bytes=771851672 weight_movement_bytes=550502400 total_nvlink_bytes=1322354072 weight_to_token_ratio=0.713223 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.628402012e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61506 payload_rows_remote=56972 dup_rows=4030 dup_rows_remote=4018 dedup_ratio=0.061493 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 22, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 4078, 5, 309, 1958, 0, 0, 582, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 1069, 0, 0, 0, 0, 0, 4092, 0] payload_rows_per_dst=[8190, 8188, 8178, 7848, 4534, 8192, 8192, 8184] dup_rows_per_dst=[0, 728, 0, 0, 12, 1142, 10, 2138] combine_rows_per_src=[8190, 8188, 8178, 7848, 4534, 8192, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=391630848 dispatch_meta_bytes=227456 combine_payload_bytes=391630848 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=391858304 ingress_bytes=721932288 link_bytes=721932288 token_bytes=783489152 weight_movement_bytes=330301440 total_nvlink_bytes=1113790592 weight_to_token_ratio=0.421578 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.255558986e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55232 payload_rows_remote=47048 dup_rows=10304 dup_rows_remote=3902 dedup_ratio=0.157227 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 19, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 4084, 4, 237, 2040, 0, 0, 678, 0, 1, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 1060, 0, 0, 0, 0, 0, 4092, 0] payload_rows_per_dst=[8190, 3066, 8192, 7932, 6770, 4706, 8192, 8184] dup_rows_per_dst=[0, 270, 0, 0, 3074, 536, 22, 6402] combine_rows_per_src=[8190, 3066, 8192, 7932, 6770, 4706, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55714 payload_rows_remote=47526 dup_rows=9822 dup_rows_remote=9806 dedup_ratio=0.149872 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 31, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 3765, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4068, 4, 170, 2534, 0, 0, 250, 0, 2, 0, 4077, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 1246, 0, 0, 0, 0, 0, 4087, 0] payload_rows_per_dst=[8188, 8190, 8178, 7748, 8188, 4584, 8146, 2492] dup_rows_per_dst=[16, 516, 48, 3582, 5402, 258, 0, 0] combine_rows_per_src=[8188, 8190, 8178, 7748, 8188, 4584, 8146, 2492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=408375296 dispatch_meta_bytes=243960 combine_payload_bytes=408375296 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=408619256 ingress_bytes=628576256 link_bytes=628576256 token_bytes=816994552 weight_movement_bytes=220200960 total_nvlink_bytes=1037195512 weight_to_token_ratio=0.269526 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.093197492e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62134 payload_rows_remote=53968 dup_rows=3402 dup_rows_remote=3354 dedup_ratio=0.051910 router_tokens_per_expert=[0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 84, 0, 4022, 1, 0, 0, 0, 0, 30, 2641, 0, 0, 0, 0, 4096, 0, 0, 0, 3872, 4096, 3560, 0, 2945, 0, 0, 0, 0, 3216, 0, 0, 0, 0, 0, 0, 63, 0, 0, 36, 7, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 37, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8166, 8182, 7120, 6432, 7756, 8094, 8192] dup_rows_per_dst=[30, 48, 3050, 0, 0, 186, 14, 74] combine_rows_per_src=[8192, 8166, 8182, 7120, 6432, 7756, 8094, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=337240064 dispatch_meta_bytes=203800 combine_payload_bytes=337240064 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=337443864 ingress_bytes=557441024 link_bytes=557441024 token_bytes=674683928 weight_movement_bytes=220200960 total_nvlink_bytes=894884888 weight_to_token_ratio=0.326376 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=21760 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.694816240e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59716 payload_rows_remote=51544 dup_rows=5820 dup_rows_remote=5820 dedup_ratio=0.088806 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 319, 12, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 3786, 0, 0, 1, 4093, 0, 0, 1, 0, 0, 4083, 3, 415, 1273, 0, 0, 1108, 0, 1, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 1306, 0, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8192, 8190, 8172, 7572, 6878, 8192, 8192, 4328] dup_rows_per_dst=[0, 638, 0, 2, 2150, 2196, 6, 828] combine_rows_per_src=[8192, 8190, 8172, 7572, 6878, 8192, 8192, 4328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=340666368 dispatch_meta_bytes=229328 combine_payload_bytes=340666368 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=340895696 ingress_bytes=560867328 link_bytes=560867328 token_bytes=681562064 weight_movement_bytes=220200960 total_nvlink_bytes=901763024 weight_to_token_ratio=0.323083 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.754405302e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62062 payload_rows_remote=53870 dup_rows=3474 dup_rows_remote=3474 dedup_ratio=0.053009 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3640, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 375, 2485, 0, 0, 315, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1143, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8182, 8192, 7280, 5642, 8190, 8192, 8192] dup_rows_per_dst=[0, 482, 0, 0, 78, 628, 0, 2286] combine_rows_per_src=[8192, 8182, 8192, 7280, 5642, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62096 payload_rows_remote=54176 dup_rows=3440 dup_rows_remote=3440 dedup_ratio=0.052490 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 11, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 0, 253, 2275, 0, 0, 414, 0, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 1131, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8190, 8190, 7920, 5040, 8180, 8192, 8192] dup_rows_per_dst=[0, 360, 0, 0, 16, 800, 2, 2262] combine_rows_per_src=[8192, 8190, 8190, 7920, 5040, 8180, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64202 payload_rows_remote=56706 dup_rows=1334 dup_rows_remote=1306 dedup_ratio=0.020355 router_tokens_per_expert=[16, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3778, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 355, 0, 0, 3407, 0, 0, 0, 0, 419, 0, 0, 4091, 0, 0, 191, 0, 0, 0, 0, 10, 0] payload_rows_per_dst=[8190, 8192, 7558, 8192, 8190, 7496, 8192, 8192] dup_rows_per_dst=[20, 56, 0, 0, 0, 28, 828, 402] combine_rows_per_src=[8190, 8192, 7558, 8192, 8190, 7496, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=386842624 dispatch_meta_bytes=229288 combine_payload_bytes=386842624 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=387071912 ingress_bytes=496943104 link_bytes=496943104 token_bytes=773914536 weight_movement_bytes=110100480 total_nvlink_bytes=884015016 weight_to_token_ratio=0.142264 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.642657910e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=369467392 dispatch_meta_bytes=229456 combine_payload_bytes=369467392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369696848 ingress_bytes=479567872 link_bytes=479567872 token_bytes=739164240 weight_movement_bytes=110100480 total_nvlink_bytes=849264720 weight_to_token_ratio=0.148953 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.340474048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=386140160 dispatch_meta_bytes=229376 combine_payload_bytes=386140160 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=386369536 ingress_bytes=606341120 link_bytes=606341120 token_bytes=772509696 weight_movement_bytes=220200960 total_nvlink_bytes=992710656 weight_to_token_ratio=0.285046 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.054526934e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=406468608 dispatch_meta_bytes=232048 combine_payload_bytes=406468608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406700656 ingress_bytes=516569088 link_bytes=516569088 token_bytes=813169264 weight_movement_bytes=110100480 total_nvlink_bytes=923269744 weight_to_token_ratio=0.135397 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=24832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.983986050e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63148 payload_rows_remote=54964 dup_rows=2388 dup_rows_remote=2388 dedup_ratio=0.036438 router_tokens_per_expert=[72, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 27, 11, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3539, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 4, 0, 0, 589, 0, 0, 2932, 0, 0, 1, 0, 537, 0, 0, 4088, 0, 0, 506, 0, 0, 0, 0, 6, 0] payload_rows_per_dst=[8190, 8192, 7088, 8184, 8184, 6930, 8188, 8192] dup_rows_per_dst=[104, 76, 0, 0, 0, 120, 1064, 1024] combine_rows_per_src=[8190, 8192, 7088, 8184, 8184, 6930, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62000 payload_rows_remote=53808 dup_rows=3536 dup_rows_remote=2832 dedup_ratio=0.053955 router_tokens_per_expert=[40, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 27, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3579, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 4, 0, 0, 347, 0, 0, 3221, 0, 0, 6, 0, 646, 0, 0, 4070, 0, 0, 335, 0, 0, 0, 0, 17, 0] payload_rows_per_dst=[7314, 7668, 7166, 8184, 8180, 7108, 8188, 8192] dup_rows_per_dst=[64, 74, 0, 244, 280, 36, 2134, 704] combine_rows_per_src=[7314, 7668, 7166, 8184, 8180, 7108, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=388333568 dispatch_meta_bytes=230464 combine_payload_bytes=388333568 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=388564032 ingress_bytes=718635008 link_bytes=718635008 token_bytes=776897600 weight_movement_bytes=330301440 total_nvlink_bytes=1107199040 weight_to_token_ratio=0.425154 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249824474e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7017230760 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=3509741992 ingress_bytes=5158995968 link_bytes=5158995968 total_nvlink_bytes=8668737960 weight_to_token_ratio=0.235350 prefetch_share_of_ingress=0.213415 dedup_ratio=0.139011 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=7493 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.972342497e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50874 payload_rows_remote=42682 dup_rows=14662 dup_rows_remote=14418 dedup_ratio=0.223724 router_tokens_per_expert=[78, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 8, 33, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3792, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4080, 6, 0, 0, 462, 0, 0, 3181, 0, 0, 3, 0, 408, 0, 0, 4086, 0, 0, 288, 0, 0, 0, 0, 6, 0] payload_rows_per_dst=[8192, 8192, 8112, 8006, 7986, 8184, 816, 1386] dup_rows_per_dst=[244, 82, 6982, 0, 0, 7286, 6, 62] combine_rows_per_src=[8192, 8192, 8112, 8006, 7986, 8184, 816, 1386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=393981952 dispatch_meta_bytes=229408 combine_payload_bytes=393981952 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=394211360 ingress_bytes=724283392 link_bytes=724283392 token_bytes=788193312 weight_movement_bytes=330301440 total_nvlink_bytes=1118494752 weight_to_token_ratio=0.419061 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.259647943e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58888 payload_rows_remote=50700 dup_rows=6648 dup_rows_remote=4898 dedup_ratio=0.101440 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 863, 43, 0, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 4063, 6, 252, 1651, 0, 0, 583, 0, 0, 0, 4047, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 1033, 0, 0, 0, 0, 1, 4091, 0] payload_rows_per_dst=[8184, 8188, 8188, 7690, 8192, 8186, 8192, 2068] dup_rows_per_dst=[0, 1750, 8, 0, 3810, 1074, 6, 0] combine_rows_per_src=[8184, 8188, 8188, 7690, 8192, 8186, 8192, 2068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=385695744 dispatch_meta_bytes=226560 combine_payload_bytes=385695744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385922304 ingress_bytes=495796224 link_bytes=495796224 token_bytes=771618048 weight_movement_bytes=110100480 total_nvlink_bytes=881718528 weight_to_token_ratio=0.142688 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.622711780e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61660 payload_rows_remote=55048 dup_rows=3876 dup_rows_remote=3876 dedup_ratio=0.059143 router_tokens_per_expert=[46, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3297, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 0, 664, 0, 0, 2366, 0, 0, 2, 0, 658, 0, 0, 4063, 0, 0, 1156, 0, 0, 0, 0, 40, 0] payload_rows_per_dst=[8190, 8192, 6612, 8174, 8172, 5950, 8178, 8192] dup_rows_per_dst=[54, 52, 0, 0, 0, 110, 1268, 2392] combine_rows_per_src=[8190, 8192, 6612, 8174, 8172, 5950, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64248 payload_rows_remote=56062 dup_rows=1288 dup_rows_remote=792 dedup_ratio=0.019653 router_tokens_per_expert=[104, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3771, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 406, 0, 0, 3395, 0, 0, 0, 0, 248, 0, 0, 4093, 0, 0, 253, 0, 0, 0, 0, 8, 0] payload_rows_per_dst=[8192, 8192, 7542, 8192, 8192, 7560, 8186, 8192] dup_rows_per_dst=[194, 34, 0, 0, 0, 42, 496, 522] combine_rows_per_src=[8192, 8192, 7542, 8192, 8192, 7560, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=305944576 dispatch_meta_bytes=228400 combine_payload_bytes=305944576 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=306172976 ingress_bytes=526145536 link_bytes=526145536 token_bytes=612117552 weight_movement_bytes=220200960 total_nvlink_bytes=832318512 weight_to_token_ratio=0.359736 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.150536232e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7087082888 prefetch_bytes=1211105280 reduce_bytes=1541406720 weight_movement_bytes=2752512000 egress_bytes=3544685960 ingress_bytes=6294908928 link_bytes=6294908928 total_nvlink_bytes=9839594888 weight_to_token_ratio=0.388384 prefetch_share_of_ingress=0.192394 dedup_ratio=0.140402 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=14 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047426 gemm_inner_pad_upper_bound=7366 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.094788196e-04 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6417312896 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3209733248 ingress_bytes=5079287808 link_bytes=5079287808 total_nvlink_bytes=8289021056 weight_to_token_ratio=0.291665 prefetch_share_of_ingress=0.195087 dedup_ratio=0.197131 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=7239 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.833716897e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=363417600 dispatch_meta_bytes=222392 combine_payload_bytes=363417600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=363639992 ingress_bytes=473518080 link_bytes=473518080 token_bytes=727057592 weight_movement_bytes=110100480 total_nvlink_bytes=837158072 weight_to_token_ratio=0.151433 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.235258215e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63488 payload_rows_remote=55308 dup_rows=2048 dup_rows_remote=2048 dedup_ratio=0.031250 router_tokens_per_expert=[84, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 9, 10, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3661, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 1, 0, 0, 418, 0, 0, 3119, 0, 0, 1, 0, 493, 0, 0, 4074, 0, 0, 436, 0, 0, 0, 0, 9, 0] payload_rows_per_dst=[8192, 8192, 7334, 8180, 8180, 7032, 8186, 8192] dup_rows_per_dst=[126, 38, 0, 0, 0, 44, 950, 890] combine_rows_per_src=[8192, 8192, 7334, 8180, 8180, 7032, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=394584064 dispatch_meta_bytes=235696 combine_payload_bytes=394584064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394819760 ingress_bytes=504684544 link_bytes=504684544 token_bytes=789403824 weight_movement_bytes=110100480 total_nvlink_bytes=899504304 weight_to_token_ratio=0.139473 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.777294284e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=401852416 dispatch_meta_bytes=227416 combine_payload_bytes=401852416 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=402079832 ingress_bytes=622053376 link_bytes=622053376 token_bytes=803932248 weight_movement_bytes=220200960 total_nvlink_bytes=1024133208 weight_to_token_ratio=0.273905 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.081853131e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6280389584 prefetch_bytes=1211105280 reduce_bytes=1101004800 weight_movement_bytes=2312110080 egress_bytes=3141293008 ingress_bytes=5451206656 link_bytes=5451206656 total_nvlink_bytes=8592499664 weight_to_token_ratio=0.368148 prefetch_share_of_ingress=0.222172 dedup_ratio=0.215155 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=10 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6731 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.480544944e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6817778296 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3410010744 ingress_bytes=5389576192 link_bytes=5389576192 total_nvlink_bytes=8799586936 weight_to_token_ratio=0.290682 prefetch_share_of_ingress=0.204284 dedup_ratio=0.160553 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.373359431e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6988605648 prefetch_bytes=880803840 reduce_bytes=1321205760 weight_movement_bytes=2202009600 egress_bytes=3495438544 ingress_bytes=5695176704 link_bytes=5695176704 total_nvlink_bytes=9190615248 weight_to_token_ratio=0.315086 prefetch_share_of_ingress=0.154658 dedup_ratio=0.144162 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=12 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6731 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.904848984e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=396447744 dispatch_meta_bytes=229424 combine_payload_bytes=396447744 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=396677168 ingress_bytes=726749184 link_bytes=726749184 token_bytes=793124912 weight_movement_bytes=330301440 total_nvlink_bytes=1123426352 weight_to_token_ratio=0.416456 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.263936361e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63230 payload_rows_remote=55038 dup_rows=2306 dup_rows_remote=2194 dedup_ratio=0.035187 router_tokens_per_expert=[151, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 18, 38, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3563, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 4, 0, 0, 352, 0, 0, 3190, 0, 0, 7, 0, 628, 0, 0, 4052, 0, 0, 366, 0, 0, 0, 0, 39, 0] payload_rows_per_dst=[8192, 8192, 7142, 8180, 8172, 7332, 7828, 8192] dup_rows_per_dst=[78, 112, 0, 0, 0, 278, 1028, 810] combine_rows_per_src=[8192, 8192, 7142, 8180, 8172, 7332, 7828, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7145652128 prefetch_bytes=880803840 reduce_bytes=1651507200 weight_movement_bytes=2532311040 egress_bytes=3573981088 ingress_bytes=6103982080 link_bytes=6103982080 total_nvlink_bytes=9677963168 weight_to_token_ratio=0.354385 prefetch_share_of_ingress=0.144300 dedup_ratio=0.133484 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=15 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.061582877e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=394512384 dispatch_meta_bytes=228928 combine_payload_bytes=394512384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394741312 ingress_bytes=504612864 link_bytes=504612864 token_bytes=789253696 weight_movement_bytes=110100480 total_nvlink_bytes=899354176 weight_to_token_ratio=0.139499 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.776047651e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6726550720 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=3364400320 ingress_bytes=5013657600 link_bytes=5013657600 total_nvlink_bytes=8378057920 weight_to_token_ratio=0.245521 prefetch_share_of_ingress=0.219601 dedup_ratio=0.164230 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051471 gemm_inner_pad_upper_bound=6096 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.719575171e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none + [2026-08-26 18:09:54] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 717.2 | throughput per GPU (TFLOP/s/GPU): 172.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.542791E+00 | loss scale: 1.0 | grad norm: 4.301 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38502 dup_rows=21170 dup_rows_remote=18678 dedup_ratio=0.323029 router_tokens_per_expert=[754, 1050, 260, 515, 1001, 658, 906, 241, 368, 482, 690, 224, 4, 994, 473, 524, 891, 538, 112, 88, 759, 788, 671, 382, 450, 941, 424, 625, 862, 364, 558, 21, 669, 658, 1077, 13, 778, 25, 966, 192, 393, 749, 336, 746, 1, 790, 183, 576, 343, 213, 737, 328, 380, 835, 230, 507, 994, 716, 474, 217, 473, 312, 203, 36] payload_rows_per_dst=[5230, 5852, 6470, 5754, 4456, 5358, 5382, 5864] dup_rows_per_dst=[2124, 2982, 4088, 2736, 1630, 2678, 2440, 2492] combine_rows_per_src=[5230, 5852, 6470, 5754, 4456, 5358, 5382, 5864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44746 payload_rows_remote=39308 dup_rows=20790 dup_rows_remote=18352 dedup_ratio=0.317230 router_tokens_per_expert=[587, 1005, 337, 202, 572, 1118, 668, 73, 905, 507, 626, 362, 18, 541, 184, 795, 778, 361, 89, 103, 576, 365, 613, 292, 425, 643, 412, 745, 814, 402, 740, 18, 791, 570, 793, 30, 645, 1, 704, 392, 751, 674, 563, 769, 1, 1029, 169, 604, 555, 435, 442, 497, 335, 669, 194, 726, 831, 1318, 318, 564, 398, 513, 586, 25] payload_rows_per_dst=[6006, 5438, 5002, 5914, 5248, 5904, 5172, 6062] dup_rows_per_dst=[3118, 2438, 1352, 2484, 2604, 3216, 2534, 3044] combine_rows_per_src=[6006, 5438, 5002, 5914, 5248, 5904, 5172, 6062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38858 dup_rows=21158 dup_rows_remote=18588 dedup_ratio=0.322845 router_tokens_per_expert=[1128, 734, 458, 763, 638, 725, 401, 87, 252, 443, 699, 444, 64, 686, 237, 697, 1011, 563, 83, 66, 614, 513, 577, 381, 454, 451, 506, 494, 829, 398, 739, 30, 412, 711, 384, 27, 863, 7, 1119, 435, 547, 568, 475, 689, 9, 732, 176, 849, 656, 477, 686, 596, 418, 946, 236, 565, 779, 692, 598, 321, 354, 843, 427, 6] payload_rows_per_dst=[4934, 5644, 6006, 5594, 5284, 5520, 5852, 5544] dup_rows_per_dst=[2016, 2850, 3078, 2208, 2632, 2570, 3308, 2496] combine_rows_per_src=[4934, 5644, 6006, 5594, 5284, 5520, 5852, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44038 payload_rows_remote=38526 dup_rows=21498 dup_rows_remote=19488 dedup_ratio=0.328033 router_tokens_per_expert=[1014, 1174, 466, 192, 378, 678, 552, 81, 430, 406, 203, 233, 46, 839, 100, 810, 651, 756, 30, 92, 554, 645, 531, 172, 340, 388, 528, 572, 829, 399, 679, 26, 800, 963, 437, 20, 837, 71, 800, 675, 488, 589, 467, 754, 6, 981, 145, 599, 584, 548, 359, 1125, 393, 642, 166, 685, 1151, 788, 432, 424, 524, 1240, 269, 12] payload_rows_per_dst=[4726, 4454, 6130, 5512, 5850, 5356, 5678, 6332] dup_rows_per_dst=[1996, 1680, 3080, 2010, 3356, 2702, 3326, 3348] combine_rows_per_src=[4726, 4454, 6130, 5512, 5850, 5356, 5678, 6332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43360 payload_rows_remote=38476 dup_rows=22176 dup_rows_remote=20048 dedup_ratio=0.338379 router_tokens_per_expert=[363, 917, 652, 953, 123, 1149, 865, 82, 857, 708, 853, 593, 22, 979, 285, 1057, 784, 432, 21, 46, 744, 205, 660, 321, 309, 383, 583, 545, 654, 412, 308, 7, 901, 1373, 570, 54, 1138, 0, 863, 413, 1069, 459, 375, 424, 24, 485, 395, 705, 558, 325, 153, 774, 161, 548, 160, 827, 545, 554, 454, 178, 245, 581, 572, 13] payload_rows_per_dst=[4558, 7118, 5666, 4820, 6318, 5446, 4884, 4550] dup_rows_per_dst=[1518, 5888, 2594, 1582, 4306, 2426, 2128, 1734] combine_rows_per_src=[4558, 7118, 5666, 4820, 6318, 5446, 4884, 4550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43934 payload_rows_remote=38466 dup_rows=21602 dup_rows_remote=18544 dedup_ratio=0.329620 router_tokens_per_expert=[609, 1228, 392, 515, 297, 1000, 1156, 206, 389, 534, 444, 270, 14, 862, 205, 531, 450, 547, 33, 71, 871, 157, 513, 313, 641, 506, 559, 830, 774, 293, 615, 22, 667, 611, 743, 29, 800, 0, 1136, 277, 564, 603, 450, 619, 3, 536, 531, 460, 436, 476, 747, 771, 395, 776, 614, 433, 969, 1190, 633, 187, 336, 482, 439, 8] payload_rows_per_dst=[5278, 5006, 5684, 5802, 5468, 5260, 5882, 5554] dup_rows_per_dst=[2218, 2346, 2682, 2678, 3058, 2272, 3414, 2934] combine_rows_per_src=[5278, 5006, 5684, 5802, 5468, 5260, 5882, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43680 payload_rows_remote=37212 dup_rows=21856 dup_rows_remote=16960 dedup_ratio=0.333496 router_tokens_per_expert=[783, 825, 549, 379, 1037, 1057, 705, 91, 342, 505, 318, 294, 17, 626, 173, 564, 545, 377, 24, 86, 742, 518, 693, 383, 405, 947, 521, 885, 678, 298, 735, 48, 1058, 622, 655, 8, 743, 9, 708, 404, 435, 865, 446, 697, 0, 801, 155, 625, 639, 203, 653, 658, 213, 684, 202, 553, 983, 1163, 587, 213, 412, 861, 354, 9] payload_rows_per_dst=[6468, 4226, 5144, 6200, 4986, 5536, 5200, 5920] dup_rows_per_dst=[4896, 1452, 1592, 3352, 2398, 2512, 2410, 3244] combine_rows_per_src=[6468, 4226, 5144, 6200, 4986, 5536, 5200, 5920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275982336 dispatch_meta_bytes=228720 combine_payload_bytes=275982336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276211056 ingress_bytes=386082816 link_bytes=386082816 token_bytes=552193392 weight_movement_bytes=110100480 total_nvlink_bytes=662293872 weight_to_token_ratio=0.199388 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.714615167e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281759744 dispatch_meta_bytes=230640 combine_payload_bytes=281759744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281990384 ingress_bytes=391860224 link_bytes=391860224 token_bytes=563750128 weight_movement_bytes=110100480 total_nvlink_bytes=673850608 weight_to_token_ratio=0.195300 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.815093795e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=229784 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278763928 ingress_bytes=388634624 link_bytes=388634624 token_bytes=557298072 weight_movement_bytes=110100480 total_nvlink_bytes=667398552 weight_to_token_ratio=0.197561 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758995305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276154368 dispatch_meta_bytes=232056 combine_payload_bytes=276154368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276386424 ingress_bytes=386254848 link_bytes=386254848 token_bytes=552540792 weight_movement_bytes=110100480 total_nvlink_bytes=662641272 weight_to_token_ratio=0.199262 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.717607087e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275795968 dispatch_meta_bytes=234096 combine_payload_bytes=275795968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276030064 ingress_bytes=385896448 link_bytes=385896448 token_bytes=551826032 weight_movement_bytes=110100480 total_nvlink_bytes=661926512 weight_to_token_ratio=0.199520 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.711373921e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43902 payload_rows_remote=38702 dup_rows=21634 dup_rows_remote=19866 dedup_ratio=0.330109 router_tokens_per_expert=[1048, 965, 1238, 336, 443, 807, 642, 73, 644, 552, 360, 518, 21, 755, 383, 848, 526, 539, 45, 73, 575, 522, 668, 209, 553, 698, 702, 438, 672, 622, 736, 24, 627, 695, 749, 33, 1225, 35, 273, 334, 479, 787, 698, 814, 7, 945, 110, 550, 408, 304, 711, 393, 419, 1040, 222, 466, 667, 548, 411, 173, 300, 679, 412, 19] payload_rows_per_dst=[6076, 5438, 5200, 6010, 5322, 5822, 5402, 4632] dup_rows_per_dst=[4374, 2724, 1768, 2880, 2620, 2958, 2524, 1786] combine_rows_per_src=[6076, 5438, 5200, 6010, 5322, 5822, 5402, 4632] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275724288 dispatch_meta_bytes=228040 combine_payload_bytes=275724288 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=275952328 ingress_bytes=826226688 link_bytes=826226688 token_bytes=551676616 weight_movement_bytes=550502400 total_nvlink_bytes=1102179016 weight_to_token_ratio=0.997872 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=6144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.436944101e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=266735616 dispatch_meta_bytes=216688 combine_payload_bytes=266735616 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=266952304 ingress_bytes=597037056 link_bytes=597037056 token_bytes=533687920 weight_movement_bytes=330301440 total_nvlink_bytes=863989360 weight_to_token_ratio=0.618904 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.038345636e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47332 payload_rows_remote=40024 dup_rows=18204 dup_rows_remote=14708 dedup_ratio=0.277771 router_tokens_per_expert=[0, 0, 0, 13, 137, 0, 1390, 46, 833, 701, 171, 0, 12, 3151, 0, 2882, 69, 2923, 0, 0, 152, 3105, 0, 0, 0, 0, 228, 2714, 2317, 36, 0, 378, 0, 0, 0, 0, 6, 107, 0, 0, 0, 37, 38, 8, 2, 0, 2731, 0, 0, 1028, 29, 1, 2058, 0, 0, 63, 459, 13, 0, 122, 46, 2993, 1743, 26] payload_rows_per_dst=[3158, 3104, 7892, 5260, 7800, 5554, 7256, 7308] dup_rows_per_dst=[14, 330, 4606, 658, 4492, 78, 4530, 3496] combine_rows_per_src=[3158, 3104, 7892, 5260, 7800, 5554, 7256, 7308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51262 payload_rows_remote=43936 dup_rows=14274 dup_rows_remote=11308 dedup_ratio=0.217804 router_tokens_per_expert=[0, 0, 0, 4, 116, 0, 1257, 6, 1202, 855, 209, 0, 11, 3382, 0, 2869, 56, 2374, 0, 6, 250, 3182, 1, 0, 1, 0, 278, 2829, 1513, 52, 0, 452, 0, 7, 0, 2, 6, 110, 2, 0, 0, 61, 7, 3, 0, 0, 2593, 0, 1, 1400, 27, 0, 2167, 0, 0, 64, 676, 6, 0, 130, 31, 2391, 2105, 74] payload_rows_per_dst=[6992, 7326, 5124, 6734, 6940, 7156, 5922, 5068] dup_rows_per_dst=[2138, 2966, 250, 3516, 78, 2954, 1396, 976] combine_rows_per_src=[6992, 7326, 5124, 6734, 6940, 7156, 5922, 5068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48862 payload_rows_remote=43236 dup_rows=16674 dup_rows_remote=16646 dedup_ratio=0.254425 router_tokens_per_expert=[0, 0, 0, 36, 161, 0, 1400, 1, 785, 791, 340, 0, 41, 2879, 0, 2945, 40, 2890, 0, 0, 38, 3236, 0, 0, 0, 0, 186, 2786, 1833, 3, 0, 537, 0, 0, 0, 0, 0, 270, 0, 0, 0, 40, 12, 0, 0, 0, 2775, 0, 0, 1192, 30, 0, 1796, 0, 0, 6, 581, 0, 0, 121, 56, 2510, 2438, 13] payload_rows_per_dst=[6594, 3456, 7424, 6796, 6424, 5626, 5148, 7394] dup_rows_per_dst=[2236, 458, 3726, 3596, 1388, 28, 1198, 4044] combine_rows_per_src=[6594, 3456, 7424, 6796, 6424, 5626, 5148, 7394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277415936 dispatch_meta_bytes=234272 combine_payload_bytes=277415936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277650208 ingress_bytes=387516416 link_bytes=387516416 token_bytes=555066144 weight_movement_bytes=110100480 total_nvlink_bytes=665166624 weight_to_token_ratio=0.198356 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=5760 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.739547829e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46608 payload_rows_remote=39236 dup_rows=18928 dup_rows_remote=14232 dedup_ratio=0.288818 router_tokens_per_expert=[0, 0, 0, 4, 192, 0, 1605, 15, 563, 984, 154, 0, 56, 2711, 0, 3234, 12, 2417, 0, 0, 29, 3470, 0, 0, 0, 0, 169, 2909, 1509, 3, 0, 344, 0, 0, 0, 0, 0, 72, 0, 1, 0, 17, 10, 0, 0, 0, 2643, 0, 0, 1148, 35, 0, 1927, 0, 0, 15, 807, 1, 3, 184, 70, 2431, 2988, 36] payload_rows_per_dst=[3606, 3224, 8028, 3614, 7710, 5332, 7372, 7722] dup_rows_per_dst=[26, 290, 3828, 436, 4326, 8, 4696, 5318] combine_rows_per_src=[3606, 3224, 8028, 3614, 7710, 5332, 7372, 7722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52280 payload_rows_remote=45958 dup_rows=13256 dup_rows_remote=11516 dedup_ratio=0.202271 router_tokens_per_expert=[0, 0, 0, 1, 141, 0, 1585, 0, 1070, 865, 247, 0, 51, 3351, 0, 3124, 9, 2657, 0, 2, 17, 3807, 0, 0, 3, 0, 25, 2868, 991, 4, 0, 140, 0, 0, 0, 0, 45, 75, 0, 10, 0, 17, 10, 5, 0, 0, 2706, 0, 0, 1176, 67, 0, 2260, 0, 0, 2, 464, 2, 0, 181, 58, 2297, 2401, 34] payload_rows_per_dst=[7920, 7590, 5346, 6322, 6856, 7132, 5750, 5364] dup_rows_per_dst=[3148, 3124, 24, 1740, 106, 2938, 1260, 916] combine_rows_per_src=[7920, 7590, 5346, 6322, 6856, 7132, 5750, 5364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49112 payload_rows_remote=41616 dup_rows=16424 dup_rows_remote=14038 dedup_ratio=0.250610 router_tokens_per_expert=[0, 0, 0, 5, 189, 0, 1246, 0, 1000, 1022, 99, 0, 8, 3195, 0, 3223, 10, 2653, 0, 0, 68, 3501, 0, 0, 0, 0, 55, 2762, 1986, 7, 0, 378, 0, 0, 0, 0, 13, 98, 0, 0, 0, 55, 20, 0, 0, 0, 2925, 0, 0, 1275, 46, 0, 1927, 0, 0, 24, 499, 0, 0, 104, 12, 2595, 1728, 40] payload_rows_per_dst=[7496, 7796, 5360, 5702, 6526, 7402, 5288, 3542] dup_rows_per_dst=[2386, 6538, 102, 1828, 86, 3788, 1256, 440] combine_rows_per_src=[7496, 7796, 5360, 5702, 6526, 7402, 5288, 3542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51502 payload_rows_remote=45236 dup_rows=14034 dup_rows_remote=13992 dedup_ratio=0.214142 router_tokens_per_expert=[0, 0, 0, 6, 177, 0, 1427, 1, 718, 1091, 67, 0, 16, 3079, 0, 2941, 2, 2906, 0, 0, 61, 3502, 0, 0, 0, 0, 202, 2411, 1852, 12, 0, 293, 0, 0, 0, 0, 5, 70, 0, 0, 0, 72, 7, 0, 0, 0, 3099, 0, 0, 1113, 38, 0, 2472, 0, 0, 22, 530, 2, 0, 101, 59, 2613, 1785, 16] payload_rows_per_dst=[7378, 5806, 6524, 6470, 6266, 6562, 6050, 6446] dup_rows_per_dst=[2848, 1550, 1724, 3070, 42, 1130, 1240, 2430] combine_rows_per_src=[7378, 5806, 6524, 6470, 6266, 6562, 6050, 6446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=286892032 dispatch_meta_bytes=218928 combine_payload_bytes=286892032 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=287110960 ingress_bytes=507092992 link_bytes=507092992 token_bytes=574002992 weight_movement_bytes=220200960 total_nvlink_bytes=794203952 weight_to_token_ratio=0.383623 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.819181156e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=314933248 dispatch_meta_bytes=220976 combine_payload_bytes=314933248 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=315154224 ingress_bytes=755335168 link_bytes=755335168 token_bytes=630087472 weight_movement_bytes=440401920 total_nvlink_bytes=1070489392 weight_to_token_ratio=0.698954 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.313652088e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=281243648 dispatch_meta_bytes=213872 combine_payload_bytes=281243648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281457520 ingress_bytes=391344128 link_bytes=391344128 token_bytes=562701168 weight_movement_bytes=110100480 total_nvlink_bytes=672801648 weight_to_token_ratio=0.195664 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.806118036e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=309915648 dispatch_meta_bytes=239528 combine_payload_bytes=309915648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=310155176 ingress_bytes=420016128 link_bytes=420016128 token_bytes=620070824 weight_movement_bytes=110100480 total_nvlink_bytes=730171304 weight_to_token_ratio=0.177561 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.304771274e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=329426944 dispatch_meta_bytes=229896 combine_payload_bytes=329426944 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=329656840 ingress_bytes=659728384 link_bytes=659728384 token_bytes=659083784 weight_movement_bytes=330301440 total_nvlink_bytes=989385224 weight_to_token_ratio=0.501152 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=15104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.147376166e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=298303488 dispatch_meta_bytes=222616 combine_payload_bytes=298303488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298526104 ingress_bytes=408403968 link_bytes=408403968 token_bytes=596829592 weight_movement_bytes=110100480 total_nvlink_bytes=706930072 weight_to_token_ratio=0.184476 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.102816713e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=324251648 dispatch_meta_bytes=236912 combine_payload_bytes=324251648 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=324488560 ingress_bytes=544452608 link_bytes=544452608 token_bytes=648740208 weight_movement_bytes=220200960 total_nvlink_bytes=868941168 weight_to_token_ratio=0.339429 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.468926324e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50606 payload_rows_remote=45198 dup_rows=14930 dup_rows_remote=14760 dedup_ratio=0.227814 router_tokens_per_expert=[0, 0, 0, 30, 183, 0, 1505, 0, 1269, 1105, 425, 0, 26, 2984, 0, 3164, 49, 2643, 0, 0, 97, 3080, 0, 0, 0, 0, 64, 3316, 1086, 15, 0, 416, 0, 0, 0, 0, 8, 256, 0, 0, 0, 3, 113, 0, 0, 0, 2451, 0, 0, 917, 115, 0, 2485, 0, 0, 70, 512, 0, 0, 92, 21, 2357, 1892, 19] payload_rows_per_dst=[7034, 7758, 5408, 7106, 6132, 6786, 5958, 4424] dup_rows_per_dst=[2562, 4220, 170, 2688, 364, 3062, 1216, 648] combine_rows_per_src=[7034, 7758, 5408, 7106, 6132, 6786, 5958, 4424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52306 payload_rows_remote=44118 dup_rows=13230 dup_rows_remote=5582 dedup_ratio=0.201874 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3218, 0, 0, 4004, 0, 5, 3110, 0, 0, 0, 1069, 453, 0, 560, 1, 0, 3, 2874, 0, 0, 3336, 1515, 8, 7, 255, 0, 3180, 0, 0, 0, 0, 0, 0, 0, 1197, 0, 0, 1, 0, 0, 3708, 1, 0, 4077, 0, 1, 0, 22] payload_rows_per_dst=[6696, 8008, 6438, 7384, 6822, 6374, 2396, 8188] dup_rows_per_dst=[302, 0, 8, 3002, 1774, 496, 0, 7648] combine_rows_per_src=[6696, 8008, 6438, 7384, 6822, 6374, 2396, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50088 payload_rows_remote=42022 dup_rows=15448 dup_rows_remote=15434 dedup_ratio=0.235718 router_tokens_per_expert=[2, 0, 0, 7, 3, 1, 189, 0, 0, 8, 0, 1, 0, 0, 0, 3, 2978, 0, 0, 4028, 4, 8, 3039, 3, 1, 0, 894, 461, 5, 770, 0, 0, 0, 3190, 4, 0, 3320, 1439, 7, 5, 258, 0, 2831, 2, 0, 4, 1, 0, 1, 0, 1633, 1, 0, 0, 3, 0, 3606, 0, 0, 4044, 0, 0, 0, 14] payload_rows_per_dst=[7444, 8066, 7840, 3480, 6264, 7904, 8136, 954] dup_rows_per_dst=[860, 14, 4224, 782, 1766, 4574, 3228, 0] combine_rows_per_src=[7444, 8066, 7840, 3480, 6264, 7904, 8136, 954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51346 payload_rows_remote=49386 dup_rows=14190 dup_rows_remote=14190 dedup_ratio=0.216522 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2423, 0, 0, 4054, 0, 45, 3690, 0, 0, 0, 1707, 756, 0, 514, 0, 0, 0, 2422, 0, 0, 3868, 1126, 1, 0, 245, 2, 3166, 0, 0, 0, 0, 0, 0, 0, 980, 0, 0, 0, 0, 0, 3547, 0, 0, 4093, 0, 0, 0, 1] payload_rows_per_dst=[7742, 8108, 4904, 7990, 6094, 6356, 1960, 8192] dup_rows_per_dst=[250, 0, 32, 5344, 1004, 470, 0, 7090] combine_rows_per_src=[7742, 8108, 4904, 7990, 6094, 6356, 1960, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53942 payload_rows_remote=47192 dup_rows=11594 dup_rows_remote=11162 dedup_ratio=0.176910 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 181, 0, 0, 8, 0, 0, 0, 0, 0, 0, 2565, 0, 0, 3973, 0, 19, 2823, 0, 0, 0, 2016, 639, 0, 452, 0, 0, 0, 2530, 0, 0, 3496, 1645, 3, 0, 224, 0, 3367, 0, 0, 0, 0, 0, 0, 0, 1077, 0, 0, 0, 0, 0, 3655, 0, 0, 4091, 0, 0, 0, 4] payload_rows_per_dst=[7024, 7948, 5158, 7462, 6598, 6750, 5210, 7792] dup_rows_per_dst=[330, 14, 10, 4398, 1758, 432, 692, 3960] combine_rows_per_src=[7024, 7948, 5158, 7462, 6598, 6750, 5210, 7792] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55534 payload_rows_remote=49212 dup_rows=10002 dup_rows_remote=7584 dedup_ratio=0.152618 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 148, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3356, 0, 0, 3930, 0, 20, 2704, 0, 0, 0, 944, 701, 0, 678, 4, 0, 0, 2727, 0, 0, 3349, 2060, 0, 5, 231, 1, 2854, 0, 0, 0, 0, 0, 0, 0, 1556, 0, 0, 0, 0, 0, 3409, 0, 0, 4071, 0, 0, 0, 17] payload_rows_per_dst=[6706, 7860, 6982, 6322, 6946, 5734, 8154, 6830] dup_rows_per_dst=[292, 2, 1092, 2418, 2638, 438, 3100, 22] combine_rows_per_src=[6706, 7860, 6982, 6322, 6946, 5734, 8154, 6830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45308 payload_rows_remote=37462 dup_rows=20228 dup_rows_remote=15096 dedup_ratio=0.308655 router_tokens_per_expert=[0, 0, 0, 2, 2, 0, 170, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3223, 0, 0, 4050, 0, 13, 3118, 0, 0, 0, 603, 595, 0, 448, 0, 0, 0, 2849, 0, 0, 3466, 1488, 2, 0, 343, 0, 2972, 0, 0, 0, 0, 0, 0, 0, 1712, 0, 0, 0, 0, 0, 3596, 0, 0, 4093, 0, 0, 0, 21] payload_rows_per_dst=[7846, 8134, 7940, 2824, 2980, 7356, 8186, 42] dup_rows_per_dst=[5132, 1292, 5310, 468, 0, 4602, 3424, 0] combine_rows_per_src=[7846, 8134, 7940, 2824, 2980, 7356, 8186, 42] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56750 payload_rows_remote=49500 dup_rows=8786 dup_rows_remote=5902 dedup_ratio=0.134064 router_tokens_per_expert=[0, 0, 0, 0, 6, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2626, 0, 0, 4040, 0, 13, 2858, 0, 0, 0, 904, 487, 0, 286, 0, 0, 0, 2953, 0, 0, 3631, 2079, 32, 3, 248, 0, 3304, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 3647, 0, 0, 4083, 0, 1, 0, 3] payload_rows_per_dst=[7274, 8080, 5274, 6790, 7250, 6612, 8174, 7296] dup_rows_per_dst=[374, 0, 4, 2280, 2884, 492, 2746, 6] combine_rows_per_src=[7274, 8080, 5274, 6790, 7250, 6612, 8174, 7296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=323979264 dispatch_meta_bytes=239832 combine_payload_bytes=323979264 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=324219096 ingress_bytes=544180224 link_bytes=544180224 token_bytes=648198360 weight_movement_bytes=220200960 total_nvlink_bytes=868399320 weight_to_token_ratio=0.339712 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=21504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.464189118e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=316237824 dispatch_meta_bytes=198800 combine_payload_bytes=316237824 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=316436624 ingress_bytes=756639744 link_bytes=756639744 token_bytes=632674448 weight_movement_bytes=440401920 total_nvlink_bytes=1073076368 weight_to_token_ratio=0.696096 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.315920961e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=301213696 dispatch_meta_bytes=229824 combine_payload_bytes=301213696 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=301443520 ingress_bytes=521414656 link_bytes=521414656 token_bytes=602657216 weight_movement_bytes=220200960 total_nvlink_bytes=822858176 weight_to_token_ratio=0.365383 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.068258448e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=353998848 dispatch_meta_bytes=254304 combine_payload_bytes=353998848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=354253152 ingress_bytes=464099328 link_bytes=464099328 token_bytes=708252000 weight_movement_bytes=110100480 total_nvlink_bytes=818352480 weight_to_token_ratio=0.155454 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=22272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.071450626e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=338272256 dispatch_meta_bytes=233416 combine_payload_bytes=338272256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=338505672 ingress_bytes=448372736 link_bytes=448372736 token_bytes=676777928 weight_movement_bytes=110100480 total_nvlink_bytes=786878408 weight_to_token_ratio=0.162683 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=24960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.797939325e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=352751616 dispatch_meta_bytes=227184 combine_payload_bytes=352751616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352978800 ingress_bytes=462852096 link_bytes=462852096 token_bytes=705730416 weight_movement_bytes=110100480 total_nvlink_bytes=815830896 weight_to_token_ratio=0.156009 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=14592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.049759210e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268527616 dispatch_meta_bytes=210232 combine_payload_bytes=268527616 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=268737848 ingress_bytes=488728576 link_bytes=488728576 token_bytes=537265464 weight_movement_bytes=220200960 total_nvlink_bytes=757466424 weight_to_token_ratio=0.409855 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.499793757e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=354816000 dispatch_meta_bytes=221608 combine_payload_bytes=354816000 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=355037608 ingress_bytes=685117440 link_bytes=685117440 token_bytes=709853608 weight_movement_bytes=330301440 total_nvlink_bytes=1040155048 weight_to_token_ratio=0.465309 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.191531911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51028 payload_rows_remote=42900 dup_rows=14508 dup_rows_remote=8734 dedup_ratio=0.221375 router_tokens_per_expert=[1612, 0, 1521, 0, 123, 0, 3, 4, 3, 0, 0, 2913, 4, 0, 0, 1, 0, 0, 58, 0, 0, 0, 44, 0, 0, 49, 4, 468, 0, 1, 0, 2, 0, 0, 0, 1, 2741, 894, 0, 71, 1098, 2, 4, 0, 0, 0, 4030, 0, 0, 2461, 4054, 1, 0, 5, 0, 0, 4073, 0, 0, 4, 3, 2431, 4085, 0] payload_rows_per_dst=[8186, 8128, 8172, 8122, 6420, 2204, 4926, 4870] dup_rows_per_dst=[6486, 5774, 202, 1034, 994, 4, 8, 6] combine_rows_per_src=[8186, 8128, 8172, 8122, 6420, 2204, 4926, 4870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51464 payload_rows_remote=43278 dup_rows=14072 dup_rows_remote=8896 dedup_ratio=0.214722 router_tokens_per_expert=[1423, 0, 1468, 0, 148, 0, 0, 0, 14, 0, 0, 2993, 9, 0, 0, 0, 0, 0, 57, 0, 0, 0, 36, 0, 0, 36, 0, 339, 0, 8, 1, 0, 0, 0, 0, 0, 2600, 797, 0, 192, 1631, 0, 0, 0, 0, 0, 3997, 0, 0, 2642, 4072, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 2134, 4081, 0] payload_rows_per_dst=[5306, 6018, 7270, 4074, 6204, 8088, 6318, 8186] dup_rows_per_dst=[772, 14, 164, 490, 974, 3168, 3314, 5176] combine_rows_per_src=[5306, 6018, 7270, 4074, 6204, 8088, 6318, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55408 payload_rows_remote=47944 dup_rows=10128 dup_rows_remote=7272 dedup_ratio=0.154541 router_tokens_per_expert=[0, 0, 0, 7, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2251, 0, 0, 4005, 0, 26, 2883, 0, 0, 0, 650, 819, 0, 821, 6, 0, 0, 3136, 0, 0, 3415, 2238, 5, 0, 290, 0, 3183, 0, 0, 0, 0, 0, 0, 0, 991, 0, 0, 0, 0, 0, 3729, 0, 0, 4074, 0, 0, 0, 29] payload_rows_per_dst=[6880, 8010, 7464, 3788, 7264, 6388, 8154, 7460] dup_rows_per_dst=[384, 0, 2856, 804, 3494, 558, 1976, 56] combine_rows_per_src=[6880, 8010, 7464, 3788, 7264, 6388, 8154, 7460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54534 payload_rows_remote=49284 dup_rows=11002 dup_rows_remote=11002 dedup_ratio=0.167877 router_tokens_per_expert=[1714, 0, 1427, 0, 144, 0, 0, 0, 6, 0, 0, 2497, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 58, 0, 0, 31, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 2729, 334, 0, 130, 1706, 0, 0, 0, 0, 0, 4030, 0, 0, 2625, 4028, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 2878, 4084, 0] payload_rows_per_dst=[5730, 5006, 8168, 8056, 6030, 8122, 5250, 8172] dup_rows_per_dst=[840, 0, 124, 586, 356, 3350, 0, 5746] combine_rows_per_src=[5730, 5006, 8168, 8056, 6030, 8122, 5250, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54820 payload_rows_remote=46702 dup_rows=10716 dup_rows_remote=7740 dedup_ratio=0.163513 router_tokens_per_expert=[1197, 0, 1872, 0, 144, 0, 0, 0, 0, 0, 0, 2760, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 49, 0, 0, 61, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 2375, 1198, 0, 93, 1503, 0, 0, 0, 0, 0, 4044, 0, 0, 2446, 4053, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 2533, 4089, 0] payload_rows_per_dst=[5704, 5520, 8182, 8106, 6150, 8118, 4892, 8148] dup_rows_per_dst=[722, 0, 158, 612, 1182, 2976, 0, 5066] combine_rows_per_src=[5704, 5520, 8182, 8106, 6150, 8118, 4892, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54422 payload_rows_remote=46296 dup_rows=11114 dup_rows_remote=10400 dedup_ratio=0.169586 router_tokens_per_expert=[1656, 0, 905, 0, 117, 2, 0, 0, 7, 0, 0, 3038, 11, 0, 2, 0, 0, 0, 5, 0, 0, 0, 45, 0, 0, 53, 0, 308, 0, 2, 1, 0, 0, 0, 0, 0, 2838, 968, 0, 104, 1640, 0, 0, 0, 0, 0, 4034, 0, 0, 2674, 4056, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 2164, 4083, 0] payload_rows_per_dst=[4708, 6098, 8170, 8126, 6690, 8138, 8164, 4328] dup_rows_per_dst=[652, 18, 96, 714, 1136, 3204, 5294, 0] combine_rows_per_src=[4708, 6098, 8170, 8126, 6690, 8138, 8164, 4328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=307507200 dispatch_meta_bytes=206536 combine_payload_bytes=307507200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=307713736 ingress_bytes=417607680 link_bytes=417607680 token_bytes=615220936 weight_movement_bytes=110100480 total_nvlink_bytes=725321416 weight_to_token_ratio=0.178961 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.262884402e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52194 payload_rows_remote=44004 dup_rows=13342 dup_rows_remote=8084 dedup_ratio=0.203583 router_tokens_per_expert=[1640, 0, 917, 0, 114, 0, 0, 0, 7, 0, 0, 3056, 2, 0, 0, 0, 0, 0, 15, 0, 0, 0, 59, 0, 0, 56, 0, 182, 0, 1, 0, 0, 0, 0, 0, 0, 2988, 664, 0, 61, 1144, 0, 0, 0, 0, 0, 3973, 0, 0, 3029, 3956, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 2766, 4085, 0] payload_rows_per_dst=[8190, 7998, 8176, 7926, 6514, 3782, 6058, 3550] dup_rows_per_dst=[5258, 6078, 142, 464, 912, 488, 0, 0] combine_rows_per_src=[8190, 7998, 8176, 7926, 6514, 3782, 6058, 3550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=310216704 dispatch_meta_bytes=208696 combine_payload_bytes=310216704 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=310425400 ingress_bytes=750618624 link_bytes=750618624 token_bytes=620642104 weight_movement_bytes=440401920 total_nvlink_bytes=1061044024 weight_to_token_ratio=0.709591 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=18944 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.305449243e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55480 payload_rows_remote=49136 dup_rows=10056 dup_rows_remote=9344 dedup_ratio=0.153442 router_tokens_per_expert=[1762, 0, 1580, 0, 144, 0, 0, 0, 14, 0, 0, 2892, 11, 0, 0, 0, 0, 0, 14, 0, 0, 0, 33, 0, 0, 53, 0, 246, 0, 1, 0, 0, 0, 0, 0, 0, 2910, 565, 0, 53, 1354, 0, 0, 0, 0, 0, 4017, 0, 0, 2402, 4075, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 2490, 4084, 0] payload_rows_per_dst=[5842, 5800, 8168, 8150, 6344, 8084, 7110, 5982] dup_rows_per_dst=[1130, 34, 94, 600, 712, 2658, 3414, 1414] combine_rows_per_src=[5842, 5800, 8168, 8150, 6344, 8084, 7110, 5982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=353267712 dispatch_meta_bytes=241144 combine_payload_bytes=353267712 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=353508856 ingress_bytes=573468672 link_bytes=573468672 token_bytes=706776568 weight_movement_bytes=220200960 total_nvlink_bytes=926977528 weight_to_token_ratio=0.311557 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.973563400e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=343662592 dispatch_meta_bytes=220864 combine_payload_bytes=343662592 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=343883456 ingress_bytes=784064512 link_bytes=784064512 token_bytes=687546048 weight_movement_bytes=440401920 total_nvlink_bytes=1127947968 weight_to_token_ratio=0.640542 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.363617143e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=334759936 dispatch_meta_bytes=217768 combine_payload_bytes=334759936 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=334977704 ingress_bytes=665061376 link_bytes=665061376 token_bytes=669737640 weight_movement_bytes=330301440 total_nvlink_bytes=1000039080 weight_to_token_ratio=0.493180 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=20224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.156651117e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=331849728 dispatch_meta_bytes=226784 combine_payload_bytes=331849728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=332076512 ingress_bytes=441950208 link_bytes=441950208 token_bytes=663926240 weight_movement_bytes=110100480 total_nvlink_bytes=774026720 weight_to_token_ratio=0.165832 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=30208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.686241000e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=315420672 dispatch_meta_bytes=208352 combine_payload_bytes=315420672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=315629024 ingress_bytes=425521152 link_bytes=425521152 token_bytes=631049696 weight_movement_bytes=110100480 total_nvlink_bytes=741150176 weight_to_token_ratio=0.174472 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=11904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.400512695e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=352206848 dispatch_meta_bytes=233920 combine_payload_bytes=352206848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352440768 ingress_bytes=462307328 link_bytes=462307328 token_bytes=704647616 weight_movement_bytes=110100480 total_nvlink_bytes=814748096 weight_to_token_ratio=0.156249 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=22528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.040284799e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56868 payload_rows_remote=49256 dup_rows=8668 dup_rows_remote=8668 dedup_ratio=0.132263 router_tokens_per_expert=[0, 2, 1234, 3, 2306, 0, 0, 0, 1, 2, 1, 0, 0, 0, 0, 0, 2136, 2, 1, 3483, 3, 0, 0, 4, 0, 0, 0, 470, 4, 75, 0, 0, 3500, 1733, 0, 39, 0, 1, 0, 31, 0, 4081, 15, 1, 0, 20, 2186, 0, 0, 3293, 0, 0, 1, 120, 3802, 0, 0, 0, 221, 0, 44, 3862, 91, 0] payload_rows_per_dst=[7484, 7612, 6976, 8182, 7678, 4428, 6596, 7912] dup_rows_per_dst=[3878, 0, 10, 1078, 2930, 16, 232, 524] combine_rows_per_src=[7484, 7612, 6976, 8182, 7678, 4428, 6596, 7912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47236 payload_rows_remote=39226 dup_rows=18300 dup_rows_remote=14648 dedup_ratio=0.279236 router_tokens_per_expert=[0, 24, 761, 0, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2843, 0, 0, 3448, 0, 0, 0, 0, 0, 0, 0, 625, 0, 178, 0, 0, 3678, 1858, 0, 48, 0, 0, 0, 20, 0, 4053, 40, 0, 0, 2, 1858, 0, 0, 3085, 0, 0, 0, 79, 3792, 0, 0, 0, 232, 0, 58, 3881, 110, 0] payload_rows_per_dst=[5170, 7584, 5686, 1536, 3790, 8192, 7268, 8010] dup_rows_per_dst=[590, 0, 0, 70, 62, 10610, 3316, 3652] combine_rows_per_src=[5170, 7584, 5686, 1536, 3790, 8192, 7268, 8010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51952 payload_rows_remote=43804 dup_rows=13584 dup_rows_remote=7426 dedup_ratio=0.207275 router_tokens_per_expert=[0, 10, 535, 0, 2124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2641, 0, 0, 3591, 0, 0, 0, 0, 0, 0, 0, 148, 0, 282, 0, 0, 3780, 1473, 0, 35, 0, 0, 0, 22, 0, 4080, 6, 0, 0, 0, 2589, 0, 0, 3317, 0, 0, 0, 56, 3906, 0, 0, 0, 159, 0, 79, 3932, 3, 0] payload_rows_per_dst=[5030, 7812, 5282, 6808, 3030, 7930, 8148, 7912] dup_rows_per_dst=[308, 0, 0, 686, 30, 5968, 6158, 434] combine_rows_per_src=[5030, 7812, 5282, 6808, 3030, 7930, 8148, 7912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53134 payload_rows_remote=45432 dup_rows=12402 dup_rows_remote=9174 dedup_ratio=0.189240 router_tokens_per_expert=[0, 0, 1027, 0, 2307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2154, 0, 0, 3617, 0, 0, 0, 0, 0, 0, 0, 547, 0, 253, 0, 0, 3570, 1864, 0, 15, 0, 0, 0, 17, 0, 4094, 26, 0, 0, 3, 1819, 0, 0, 3176, 0, 0, 0, 101, 3760, 0, 0, 0, 160, 0, 279, 3936, 43, 0] payload_rows_per_dst=[5642, 7520, 4308, 8190, 3772, 7702, 8086, 7914] dup_rows_per_dst=[1026, 0, 0, 1598, 20, 3228, 5608, 922] combine_rows_per_src=[5642, 7520, 4308, 8190, 3772, 7702, 8086, 7914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51136 payload_rows_remote=43024 dup_rows=14400 dup_rows_remote=14262 dedup_ratio=0.219727 router_tokens_per_expert=[788, 0, 1120, 0, 199, 0, 0, 0, 1, 0, 0, 3295, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 65, 0, 0, 74, 0, 97, 0, 1, 0, 0, 0, 0, 0, 0, 3308, 1412, 0, 59, 1576, 0, 0, 0, 0, 0, 3909, 0, 0, 2224, 4066, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 2446, 4055, 0] payload_rows_per_dst=[4572, 7206, 8112, 8136, 8014, 3152, 7052, 4892] dup_rows_per_dst=[1438, 1844, 138, 340, 6904, 0, 3736, 0] combine_rows_per_src=[4572, 7206, 8112, 8136, 8014, 3152, 7052, 4892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=353067008 dispatch_meta_bytes=231696 combine_payload_bytes=353067008 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=353298704 ingress_bytes=573267968 link_bytes=573267968 token_bytes=706365712 weight_movement_bytes=220200960 total_nvlink_bytes=926566672 weight_to_token_ratio=0.311738 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.970072827e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55990 payload_rows_remote=47814 dup_rows=9546 dup_rows_remote=8556 dedup_ratio=0.145660 router_tokens_per_expert=[0, 9, 687, 0, 3001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1993, 0, 0, 3817, 0, 0, 0, 0, 0, 0, 0, 377, 0, 129, 0, 0, 3888, 1541, 0, 87, 0, 0, 0, 31, 3, 4077, 33, 0, 0, 0, 1925, 0, 0, 2857, 0, 0, 0, 56, 3908, 0, 0, 0, 287, 0, 50, 3933, 79, 0] payload_rows_per_dst=[6538, 7816, 7922, 8176, 7964, 3888, 5734, 7952] dup_rows_per_dst=[856, 0, 3698, 990, 3130, 34, 92, 746] combine_rows_per_src=[6538, 7816, 7922, 8176, 7964, 3888, 5734, 7952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47568 payload_rows_remote=40104 dup_rows=17968 dup_rows_remote=13826 dedup_ratio=0.274170 router_tokens_per_expert=[0, 9, 1004, 0, 2828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1962, 0, 0, 3461, 0, 0, 0, 0, 0, 0, 0, 180, 0, 174, 0, 0, 3941, 1868, 0, 47, 0, 0, 0, 18, 0, 4087, 10, 0, 0, 0, 2227, 0, 0, 2591, 0, 0, 0, 84, 3988, 0, 0, 0, 140, 0, 83, 3957, 109, 0] payload_rows_per_dst=[7464, 8106, 8072, 8192, 8068, 4472, 1704, 1490] dup_rows_per_dst=[4142, 3494, 5886, 690, 3680, 2, 22, 52] combine_rows_per_src=[7464, 8106, 8072, 8192, 8068, 4472, 1704, 1490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281171968 dispatch_meta_bytes=215496 combine_payload_bytes=281171968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=281387464 ingress_bytes=501372928 link_bytes=501372928 token_bytes=562559432 weight_movement_bytes=220200960 total_nvlink_bytes=782760392 weight_to_token_ratio=0.391427 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.719699835e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=313987072 dispatch_meta_bytes=204920 combine_payload_bytes=313987072 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=314191992 ingress_bytes=644288512 link_bytes=644288512 token_bytes=628179064 weight_movement_bytes=330301440 total_nvlink_bytes=958480504 weight_to_token_ratio=0.525808 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=22528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.120523690e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55308 payload_rows_remote=47506 dup_rows=10228 dup_rows_remote=7098 dedup_ratio=0.156067 router_tokens_per_expert=[0, 11, 583, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2282, 0, 0, 3830, 0, 0, 0, 0, 0, 0, 0, 488, 0, 261, 0, 0, 3777, 1635, 0, 74, 0, 0, 0, 40, 0, 4076, 6, 0, 0, 26, 1822, 0, 0, 2929, 0, 0, 0, 57, 3934, 0, 0, 0, 189, 0, 85, 3921, 102, 0] payload_rows_per_dst=[5904, 7868, 7608, 8190, 7802, 4106, 5928, 7902] dup_rows_per_dst=[564, 0, 3822, 1460, 3130, 396, 164, 692] combine_rows_per_src=[5904, 7868, 7608, 8190, 7802, 4106, 5928, 7902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=325656576 dispatch_meta_bytes=218424 combine_payload_bytes=325656576 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=325875000 ingress_bytes=545857536 link_bytes=545857536 token_bytes=651531576 weight_movement_bytes=220200960 total_nvlink_bytes=871732536 weight_to_token_ratio=0.337974 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.493360332e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=308396032 dispatch_meta_bytes=229144 combine_payload_bytes=308396032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=308625176 ingress_bytes=418496512 link_bytes=418496512 token_bytes=617021208 weight_movement_bytes=110100480 total_nvlink_bytes=727121688 weight_to_token_ratio=0.178439 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.278342652e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=342730752 dispatch_meta_bytes=225480 combine_payload_bytes=342730752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=342956232 ingress_bytes=452831232 link_bytes=452831232 token_bytes=685686984 weight_movement_bytes=110100480 total_nvlink_bytes=795787464 weight_to_token_ratio=0.160570 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.875479904e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=287465472 dispatch_meta_bytes=215720 combine_payload_bytes=287465472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=287681192 ingress_bytes=397565952 link_bytes=397565952 token_bytes=575146664 weight_movement_bytes=110100480 total_nvlink_bytes=685247144 weight_to_token_ratio=0.191430 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.914325789e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=340523008 dispatch_meta_bytes=218416 combine_payload_bytes=340523008 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=340741424 ingress_bytes=560723968 link_bytes=560723968 token_bytes=681264432 weight_movement_bytes=220200960 total_nvlink_bytes=901465392 weight_to_token_ratio=0.323224 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=18816 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.751912036e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59834 payload_rows_remote=51642 dup_rows=5702 dup_rows_remote=2616 dedup_ratio=0.087006 router_tokens_per_expert=[3, 438, 0, 1, 1, 0, 4055, 4091, 0, 0, 0, 0, 0, 4, 0, 1543, 1, 4, 3, 3881, 3, 1, 4091, 0, 0, 2, 0, 116, 0, 0, 0, 1, 5, 87, 0, 1, 0, 0, 0, 0, 0, 0, 3, 1, 3281, 0, 1, 0, 485, 3, 2, 3924, 118, 4, 0, 0, 2, 0, 0, 4092, 2519, 0, 1, 0] payload_rows_per_dst=[8156, 8192, 7770, 8186, 8192, 6570, 7724, 5044] dup_rows_per_dst=[1162, 3086, 16, 234, 176, 2, 1026, 0] combine_rows_per_src=[8156, 8192, 7770, 8186, 8192, 6570, 7724, 5044] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52306 payload_rows_remote=44114 dup_rows=13230 dup_rows_remote=6620 dedup_ratio=0.201874 router_tokens_per_expert=[0, 690, 0, 0, 0, 2, 3981, 4096, 0, 0, 0, 0, 0, 0, 0, 1303, 0, 35, 3, 3820, 0, 0, 4096, 1, 0, 0, 0, 124, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2700, 0, 0, 0, 439, 0, 0, 3879, 114, 0, 0, 0, 0, 0, 0, 4096, 3305, 0, 0, 0] payload_rows_per_dst=[1380, 2606, 7658, 8192, 8192, 8164, 7922, 8192] dup_rows_per_dst=[4, 0, 60, 248, 166, 5200, 942, 6610] combine_rows_per_src=[1380, 2606, 7658, 8192, 8192, 8164, 7922, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52942 payload_rows_remote=44856 dup_rows=12594 dup_rows_remote=12012 dedup_ratio=0.192169 router_tokens_per_expert=[0, 456, 0, 0, 0, 1, 4040, 4096, 0, 0, 0, 0, 0, 0, 0, 1841, 0, 7, 2, 3748, 0, 0, 4096, 1, 0, 0, 0, 135, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2936, 4, 0, 0, 190, 0, 0, 4010, 134, 0, 0, 0, 0, 0, 0, 4096, 2927, 0, 0, 0] payload_rows_per_dst=[914, 3682, 7496, 8192, 8192, 8188, 8086, 8192] dup_rows_per_dst=[0, 0, 20, 270, 94, 5774, 582, 5854] combine_rows_per_src=[914, 3682, 7496, 8192, 8192, 8188, 8086, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56754 payload_rows_remote=49850 dup_rows=8782 dup_rows_remote=7922 dedup_ratio=0.134003 router_tokens_per_expert=[0, 439, 0, 0, 0, 1, 4020, 4096, 0, 0, 0, 0, 0, 0, 0, 1214, 0, 24, 0, 3869, 0, 0, 4096, 1, 0, 0, 0, 131, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3295, 0, 0, 0, 339, 0, 0, 3986, 131, 0, 0, 1, 0, 0, 0, 4096, 2951, 0, 0, 0] payload_rows_per_dst=[7034, 2428, 7740, 8192, 8192, 6904, 8072, 8192] dup_rows_per_dst=[714, 0, 48, 262, 154, 860, 842, 5902] combine_rows_per_src=[7034, 2428, 7740, 8192, 8192, 6904, 8072, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=370169856 dispatch_meta_bytes=217032 combine_payload_bytes=370169856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370386888 ingress_bytes=480270336 link_bytes=480270336 token_bytes=740556744 weight_movement_bytes=110100480 total_nvlink_bytes=850657224 weight_to_token_ratio=0.148673 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=20352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.352691052e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55206 payload_rows_remote=49450 dup_rows=10330 dup_rows_remote=9820 dedup_ratio=0.157623 router_tokens_per_expert=[0, 2, 883, 0, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2451, 0, 0, 2781, 0, 0, 0, 0, 0, 0, 0, 495, 0, 264, 0, 0, 3651, 1429, 0, 121, 0, 0, 0, 24, 0, 4086, 54, 0, 0, 64, 2164, 0, 0, 3081, 0, 0, 0, 159, 3862, 0, 0, 0, 497, 0, 162, 3947, 105, 0] payload_rows_per_dst=[7268, 7724, 5756, 8186, 7684, 4470, 6198, 7920] dup_rows_per_dst=[3672, 0, 510, 1504, 2766, 94, 282, 1502] combine_rows_per_src=[7268, 7724, 5756, 8186, 7684, 4470, 6198, 7920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47306 payload_rows_remote=39114 dup_rows=18230 dup_rows_remote=8158 dedup_ratio=0.278168 router_tokens_per_expert=[0, 590, 0, 0, 0, 4, 4050, 4096, 0, 0, 0, 0, 0, 0, 0, 1987, 0, 33, 3, 3840, 0, 0, 4096, 1, 0, 0, 0, 115, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2803, 0, 0, 0, 569, 0, 0, 3983, 112, 0, 0, 0, 0, 0, 0, 4096, 2339, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7874, 6240, 7208, 5606, 3486, 508] dup_rows_per_dst=[10072, 3974, 3736, 176, 76, 0, 196, 0] combine_rows_per_src=[8192, 8192, 7874, 6240, 7208, 5606, 3486, 508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59550 payload_rows_remote=51358 dup_rows=5986 dup_rows_remote=5748 dedup_ratio=0.091339 router_tokens_per_expert=[0, 650, 0, 0, 0, 5, 4039, 4096, 0, 0, 0, 0, 0, 0, 0, 1751, 0, 7, 5, 3861, 0, 0, 4095, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2833, 0, 0, 0, 386, 0, 0, 3881, 120, 0, 0, 0, 0, 0, 0, 4096, 2770, 0, 0, 0] payload_rows_per_dst=[8120, 8192, 7726, 8192, 8192, 5666, 7922, 5540] dup_rows_per_dst=[1268, 3502, 20, 238, 106, 0, 852, 0] combine_rows_per_src=[8120, 8192, 7726, 8192, 8192, 5666, 7922, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59572 payload_rows_remote=51380 dup_rows=5964 dup_rows_remote=5864 dedup_ratio=0.091003 router_tokens_per_expert=[0, 856, 0, 0, 0, 3, 4061, 4096, 0, 0, 0, 0, 0, 0, 0, 1310, 0, 11, 3, 3841, 0, 0, 4095, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3053, 0, 0, 0, 517, 0, 0, 3934, 121, 0, 0, 0, 0, 0, 0, 4096, 2589, 0, 0, 0] payload_rows_per_dst=[8164, 7892, 7686, 8190, 8192, 6110, 8016, 5322] dup_rows_per_dst=[1676, 2456, 24, 260, 100, 0, 1128, 320] combine_rows_per_src=[8164, 7892, 7686, 8190, 8192, 6110, 8016, 5322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=316209152 dispatch_meta_bytes=202936 combine_payload_bytes=316209152 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=316412088 ingress_bytes=646510592 link_bytes=646510592 token_bytes=632621240 weight_movement_bytes=330301440 total_nvlink_bytes=962922680 weight_to_token_ratio=0.522116 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=20352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.124388252e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=321527808 dispatch_meta_bytes=227472 combine_payload_bytes=321527808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=321755280 ingress_bytes=541728768 link_bytes=541728768 token_bytes=643283088 weight_movement_bytes=220200960 total_nvlink_bytes=863484048 weight_to_token_ratio=0.342308 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.421554266e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=357324800 dispatch_meta_bytes=231088 combine_payload_bytes=357324800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=357555888 ingress_bytes=467425280 link_bytes=467425280 token_bytes=714880688 weight_movement_bytes=110100480 total_nvlink_bytes=824981168 weight_to_token_ratio=0.154012 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.129294402e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=354457600 dispatch_meta_bytes=237080 combine_payload_bytes=354457600 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=354694680 ingress_bytes=684759040 link_bytes=684759040 token_bytes=709152280 weight_movement_bytes=330301440 total_nvlink_bytes=1039453720 weight_to_token_ratio=0.465769 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.190908594e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280369152 dispatch_meta_bytes=189088 combine_payload_bytes=280369152 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280558240 ingress_bytes=610670592 link_bytes=610670592 token_bytes=560927392 weight_movement_bytes=330301440 total_nvlink_bytes=891228832 weight_to_token_ratio=0.588849 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062056598e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=368134144 dispatch_meta_bytes=228424 combine_payload_bytes=368134144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368362568 ingress_bytes=478234624 link_bytes=478234624 token_bytes=736496712 weight_movement_bytes=110100480 total_nvlink_bytes=846597192 weight_to_token_ratio=0.149492 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.317286672e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=368291840 dispatch_meta_bytes=228976 combine_payload_bytes=368291840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368520816 ingress_bytes=478392320 link_bytes=478392320 token_bytes=736812656 weight_movement_bytes=110100480 total_nvlink_bytes=846913136 weight_to_token_ratio=0.149428 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.320029265e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53108 payload_rows_remote=44918 dup_rows=12428 dup_rows_remote=8838 dedup_ratio=0.189636 router_tokens_per_expert=[0, 353, 0, 0, 0, 191, 0, 0, 0, 0, 3, 0, 1, 0, 0, 1794, 1, 0, 0, 0, 0, 2, 0, 574, 0, 4092, 0, 0, 0, 0, 1510, 2, 0, 1, 5, 3772, 0, 0, 4, 0, 0, 0, 0, 3, 2, 0, 4070, 5, 1, 2, 4081, 0, 0, 46, 4091, 4088, 0, 4065, 0, 0, 5, 4, 0, 0] payload_rows_per_dst=[8190, 8190, 8168, 3126, 8066, 8150, 1078, 8140] dup_rows_per_dst=[1080, 3590, 1148, 360, 6064, 170, 8, 8] combine_rows_per_src=[8190, 8190, 8168, 3126, 8066, 8150, 1078, 8140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48502 payload_rows_remote=40314 dup_rows=17034 dup_rows_remote=16784 dedup_ratio=0.259918 router_tokens_per_expert=[0, 162, 0, 0, 0, 235, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2627, 0, 0, 0, 0, 0, 0, 0, 535, 0, 4090, 0, 0, 0, 0, 983, 0, 0, 0, 0, 3637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4094, 0, 0, 24, 4096, 4091, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1714, 5262, 1492, 8192, 7274, 8192, 8188, 8188] dup_rows_per_dst=[184, 0, 106, 1954, 0, 0, 14540, 250] combine_rows_per_src=[1714, 5262, 1492, 8192, 7274, 8192, 8188, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60190 payload_rows_remote=52000 dup_rows=5346 dup_rows_remote=5188 dedup_ratio=0.081573 router_tokens_per_expert=[0, 219, 0, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 0, 0, 0, 0, 0, 0, 0, 344, 0, 4091, 0, 0, 0, 0, 1776, 0, 0, 0, 0, 3781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 2, 4096, 0, 0, 78, 4096, 4094, 0, 4079, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3518, 8192, 8192, 7562, 8186, 8190, 8158] dup_rows_per_dst=[958, 0, 688, 3542, 0, 0, 158, 0] combine_rows_per_src=[8192, 3518, 8192, 8192, 7562, 8186, 8190, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60312 payload_rows_remote=52138 dup_rows=5224 dup_rows_remote=5224 dedup_ratio=0.079712 router_tokens_per_expert=[0, 95, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1931, 0, 0, 0, 0, 0, 0, 0, 670, 0, 4096, 0, 0, 0, 0, 1586, 0, 0, 0, 0, 3673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 2, 4096, 0, 0, 45, 4096, 4090, 0, 4084, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3862, 8192, 8192, 7346, 8174, 8186, 8168] dup_rows_per_dst=[624, 0, 1340, 3172, 0, 0, 88, 0] combine_rows_per_src=[8192, 3862, 8192, 8192, 7346, 8174, 8186, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=321972224 dispatch_meta_bytes=215024 combine_payload_bytes=321972224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=322187248 ingress_bytes=432072704 link_bytes=432072704 token_bytes=644159472 weight_movement_bytes=110100480 total_nvlink_bytes=754259952 weight_to_token_ratio=0.170921 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.514454960e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60420 payload_rows_remote=52802 dup_rows=5116 dup_rows_remote=5072 dedup_ratio=0.078064 router_tokens_per_expert=[0, 396, 0, 0, 0, 0, 3944, 4095, 0, 0, 0, 0, 0, 0, 0, 1571, 0, 23, 1, 3807, 0, 0, 4096, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3244, 2, 0, 0, 441, 0, 0, 3793, 136, 0, 0, 0, 0, 0, 0, 4096, 2930, 0, 0, 0] payload_rows_per_dst=[7978, 8192, 7618, 8192, 8192, 6492, 7896, 5860] dup_rows_per_dst=[702, 3142, 44, 272, 112, 0, 844, 0] combine_rows_per_src=[7978, 8192, 7618, 8192, 8192, 6492, 7896, 5860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56590 payload_rows_remote=48398 dup_rows=8946 dup_rows_remote=7886 dedup_ratio=0.136505 router_tokens_per_expert=[0, 355, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2723, 0, 0, 0, 0, 0, 0, 0, 269, 0, 4096, 0, 0, 0, 0, 953, 0, 0, 0, 0, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 0, 4096, 0, 0, 34, 4096, 4089, 0, 4067, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8182, 7448, 8180, 70, 8134] dup_rows_per_dst=[1060, 5446, 538, 1902, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8182, 7448, 8180, 70, 8134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=288970752 dispatch_meta_bytes=228392 combine_payload_bytes=288970752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=289199144 ingress_bytes=399071232 link_bytes=399071232 token_bytes=578169896 weight_movement_bytes=110100480 total_nvlink_bytes=688270376 weight_to_token_ratio=0.190429 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.940505084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60358 payload_rows_remote=52842 dup_rows=5178 dup_rows_remote=5178 dedup_ratio=0.079010 router_tokens_per_expert=[0, 314, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1882, 0, 0, 0, 0, 0, 0, 0, 530, 0, 4096, 0, 0, 0, 0, 1468, 0, 0, 0, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4, 2, 4096, 0, 0, 41, 4096, 4094, 0, 4086, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3764, 8192, 8192, 7516, 8142, 8188, 8172] dup_rows_per_dst=[1088, 0, 1060, 2936, 0, 0, 94, 0] combine_rows_per_src=[8192, 3764, 8192, 8192, 7516, 8142, 8188, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59322 payload_rows_remote=56228 dup_rows=6214 dup_rows_remote=6012 dedup_ratio=0.094818 router_tokens_per_expert=[0, 149, 0, 0, 0, 209, 0, 3, 20, 0, 0, 4, 0, 0, 0, 2286, 0, 0, 0, 0, 0, 0, 0, 434, 0, 4096, 0, 0, 0, 0, 1334, 0, 0, 0, 0, 3722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 5, 5, 4091, 0, 0, 69, 4096, 4095, 0, 4072, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7920, 8182, 3094, 7444, 8156, 8190, 8144] dup_rows_per_dst=[722, 4264, 868, 202, 0, 0, 158, 0] combine_rows_per_src=[8192, 7920, 8182, 3094, 7444, 8156, 8190, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=372736000 dispatch_meta_bytes=228752 combine_payload_bytes=372736000 prefetch_bytes=0 reduce_bytes=660602880 egress_bytes=372964752 ingress_bytes=1033338880 link_bytes=1033338880 token_bytes=745700752 weight_movement_bytes=660602880 total_nvlink_bytes=1406303632 weight_to_token_ratio=0.885882 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=24960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.797146267e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=373725184 dispatch_meta_bytes=229448 combine_payload_bytes=373725184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373954632 ingress_bytes=483825664 link_bytes=483825664 token_bytes=747679816 weight_movement_bytes=110100480 total_nvlink_bytes=857780296 weight_to_token_ratio=0.147256 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.414524053e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=378484736 dispatch_meta_bytes=231496 combine_payload_bytes=378484736 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=378716232 ingress_bytes=598685696 link_bytes=598685696 token_bytes=757200968 weight_movement_bytes=220200960 total_nvlink_bytes=977401928 weight_to_token_ratio=0.290809 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.041212892e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=346916864 dispatch_meta_bytes=225136 combine_payload_bytes=346916864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=347142000 ingress_bytes=457017344 link_bytes=457017344 token_bytes=694058864 weight_movement_bytes=110100480 total_nvlink_bytes=804159344 weight_to_token_ratio=0.158633 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.948283277e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=378771456 dispatch_meta_bytes=232080 combine_payload_bytes=378771456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=379003536 ingress_bytes=488871936 link_bytes=488871936 token_bytes=757774992 weight_movement_bytes=110100480 total_nvlink_bytes=867875472 weight_to_token_ratio=0.145294 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.502287023e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=403042304 dispatch_meta_bytes=248960 combine_payload_bytes=403042304 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=403291264 ingress_bytes=623243264 link_bytes=623243264 token_bytes=806333568 weight_movement_bytes=220200960 total_nvlink_bytes=1026534528 weight_to_token_ratio=0.273089 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.083922542e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60942 payload_rows_remote=53220 dup_rows=4594 dup_rows_remote=1978 dedup_ratio=0.070099 router_tokens_per_expert=[10, 0, 0, 0, 0, 71, 0, 264, 0, 0, 3003, 4, 2104, 54, 4, 0, 29, 0, 75, 10, 5, 0, 1, 0, 4095, 0, 0, 0, 71, 4091, 487, 4, 4050, 0, 1, 0, 0, 3186, 0, 2, 0, 0, 0, 1, 3195, 0, 0, 3841, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 1, 42, 0, 1, 0, 4, 0] payload_rows_per_dst=[7838, 7722, 8106, 8190, 6378, 6392, 8124, 8192] dup_rows_per_dst=[534, 2616, 234, 1116, 0, 0, 0, 94] combine_rows_per_src=[7838, 7722, 8106, 8190, 6378, 6392, 8124, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49188 payload_rows_remote=40996 dup_rows=16348 dup_rows_remote=16284 dedup_ratio=0.249451 router_tokens_per_expert=[0, 0, 0, 0, 0, 94, 0, 280, 0, 0, 3136, 0, 2034, 45, 0, 0, 14, 0, 93, 21, 0, 0, 0, 0, 4096, 0, 0, 0, 88, 4096, 660, 0, 4041, 0, 0, 0, 0, 2733, 0, 0, 0, 0, 0, 0, 3221, 0, 0, 3998, 1, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 32, 0, 0, 0, 0, 0] payload_rows_per_dst=[4498, 732, 7396, 4128, 7880, 8192, 8170, 8192] dup_rows_per_dst=[296, 16, 222, 230, 4578, 10392, 550, 64] combine_rows_per_src=[4498, 732, 7396, 4128, 7880, 8192, 8170, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55484 payload_rows_remote=47324 dup_rows=10052 dup_rows_remote=10052 dedup_ratio=0.153381 router_tokens_per_expert=[0, 0, 0, 0, 0, 41, 0, 322, 0, 0, 3398, 0, 2507, 29, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 115, 4096, 708, 0, 4058, 0, 0, 0, 0, 2714, 0, 0, 0, 0, 0, 0, 2557, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 17, 0, 0, 0, 0, 0] payload_rows_per_dst=[8008, 5064, 8122, 1752, 8010, 8176, 8160, 8192] dup_rows_per_dst=[670, 8, 102, 6, 4214, 5018, 0, 34] combine_rows_per_src=[8008, 5064, 8122, 1752, 8010, 8176, 8160, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57776 payload_rows_remote=53810 dup_rows=7760 dup_rows_remote=7760 dedup_ratio=0.118408 router_tokens_per_expert=[0, 0, 0, 0, 0, 56, 0, 526, 0, 0, 3301, 0, 2398, 105, 0, 0, 0, 0, 72, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 122, 4096, 857, 0, 4054, 0, 0, 0, 0, 2922, 0, 0, 0, 0, 0, 0, 1983, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 25, 0, 0, 0, 0, 0] payload_rows_per_dst=[8130, 5698, 8112, 8192, 7298, 3966, 8188, 8192] dup_rows_per_dst=[1152, 1084, 144, 1958, 3372, 0, 0, 50] combine_rows_per_src=[8130, 5698, 8112, 8192, 7298, 3966, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=381480960 dispatch_meta_bytes=220792 combine_payload_bytes=381480960 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=381701752 ingress_bytes=711782400 link_bytes=711782400 token_bytes=763182712 weight_movement_bytes=330301440 total_nvlink_bytes=1093484152 weight_to_token_ratio=0.432795 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.237906662e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59166 payload_rows_remote=53904 dup_rows=6370 dup_rows_remote=6370 dedup_ratio=0.097198 router_tokens_per_expert=[0, 0, 0, 0, 0, 52, 0, 251, 0, 0, 3204, 0, 2586, 24, 0, 0, 4, 0, 64, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 142, 4096, 728, 0, 4021, 0, 0, 0, 0, 2631, 0, 0, 0, 0, 0, 0, 2787, 0, 0, 3965, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 17, 0, 0, 0, 0, 0] payload_rows_per_dst=[8004, 7712, 8042, 8192, 5262, 5574, 8188, 8192] dup_rows_per_dst=[532, 3916, 146, 1740, 0, 0, 2, 34] combine_rows_per_src=[8004, 7712, 8042, 8192, 5262, 5574, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60058 payload_rows_remote=51990 dup_rows=5478 dup_rows_remote=4738 dedup_ratio=0.083588 router_tokens_per_expert=[0, 0, 0, 0, 0, 16, 0, 371, 0, 0, 2370, 0, 2542, 6, 0, 0, 3, 0, 86, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 80, 4096, 816, 0, 4074, 0, 0, 0, 0, 3175, 0, 0, 0, 0, 0, 0, 2983, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 10, 0, 0, 0, 0, 0] payload_rows_per_dst=[8068, 7454, 8150, 8192, 6350, 5966, 8052, 7826] dup_rows_per_dst=[740, 2382, 178, 2176, 0, 0, 0, 2] combine_rows_per_src=[8068, 7454, 8150, 8192, 6350, 5966, 8052, 7826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60236 payload_rows_remote=52044 dup_rows=5300 dup_rows_remote=4314 dedup_ratio=0.080872 router_tokens_per_expert=[0, 120, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1668, 0, 0, 0, 0, 0, 0, 0, 493, 0, 4096, 0, 0, 0, 0, 2149, 0, 0, 0, 0, 3408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 1, 4096, 0, 0, 106, 4096, 4092, 0, 4087, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 4298, 6816, 8186, 8186, 8174] dup_rows_per_dst=[764, 3336, 986, 0, 0, 0, 214, 0] combine_rows_per_src=[8192, 8192, 8192, 4298, 6816, 8186, 8186, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=293859328 dispatch_meta_bytes=229120 combine_payload_bytes=293859328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=294088448 ingress_bytes=403959808 link_bytes=403959808 token_bytes=587947776 weight_movement_bytes=110100480 total_nvlink_bytes=698048256 weight_to_token_ratio=0.187262 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.025525461e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=339218432 dispatch_meta_bytes=229504 combine_payload_bytes=339218432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=339447936 ingress_bytes=449318912 link_bytes=449318912 token_bytes=678666368 weight_movement_bytes=110100480 total_nvlink_bytes=788766848 weight_to_token_ratio=0.162231 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.814394882e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=385710080 dispatch_meta_bytes=246280 combine_payload_bytes=385710080 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=385956360 ingress_bytes=605911040 link_bytes=605911040 token_bytes=771666440 weight_movement_bytes=220200960 total_nvlink_bytes=991867400 weight_to_token_ratio=0.285358 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.053778954e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59748 payload_rows_remote=51556 dup_rows=5788 dup_rows_remote=4346 dedup_ratio=0.088318 router_tokens_per_expert=[0, 0, 0, 0, 0, 59, 0, 311, 0, 0, 3186, 0, 2440, 30, 0, 0, 10, 0, 52, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 142, 4096, 579, 0, 4060, 0, 0, 0, 0, 2975, 0, 0, 0, 0, 0, 0, 2659, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 5, 1, 0, 0, 0, 0] payload_rows_per_dst=[8110, 7772, 8126, 8192, 5950, 5318, 8088, 8192] dup_rows_per_dst=[664, 3540, 130, 1442, 0, 0, 0, 12] combine_rows_per_src=[8110, 7772, 8126, 8192, 5950, 5318, 8088, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=386383872 dispatch_meta_bytes=241096 combine_payload_bytes=386383872 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=386624968 ingress_bytes=606584832 link_bytes=606584832 token_bytes=773008840 weight_movement_bytes=220200960 total_nvlink_bytes=993209800 weight_to_token_ratio=0.284862 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.054950789e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=372664320 dispatch_meta_bytes=226912 combine_payload_bytes=372664320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372891232 ingress_bytes=482764800 link_bytes=482764800 token_bytes=745555552 weight_movement_bytes=110100480 total_nvlink_bytes=855656032 weight_to_token_ratio=0.147676 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.396073884e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=373051392 dispatch_meta_bytes=225432 combine_payload_bytes=373051392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373276824 ingress_bytes=483151872 link_bytes=483151872 token_bytes=746328216 weight_movement_bytes=110100480 total_nvlink_bytes=856428696 weight_to_token_ratio=0.147523 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=28928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.402805702e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=369553408 dispatch_meta_bytes=223608 combine_payload_bytes=369553408 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=369777016 ingress_bytes=699854848 link_bytes=699854848 token_bytes=739330424 weight_movement_bytes=330301440 total_nvlink_bytes=1069631864 weight_to_token_ratio=0.446758 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.217162687e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63964 payload_rows_remote=56302 dup_rows=1572 dup_rows_remote=1116 dedup_ratio=0.023987 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 1, 313, 3741, 0, 6, 3, 0, 0, 2, 0, 0, 0, 0, 4071, 0, 1, 222, 4091, 2, 0, 0, 1, 3, 3998, 0, 1, 3, 1, 0, 4, 5, 0, 0, 4075, 3, 0, 0, 4080, 0, 0, 245, 4091, 2, 122, 0, 0, 0, 3, 0, 3673, 0] payload_rows_per_dst=[8184, 7662, 8190, 8158, 8004, 8166, 8188, 7412] dup_rows_per_dst=[0, 456, 14, 434, 8, 10, 466, 184] combine_rows_per_src=[8184, 7662, 8190, 8158, 8004, 8166, 8188, 7412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60842 payload_rows_remote=53066 dup_rows=4694 dup_rows_remote=1070 dedup_ratio=0.071625 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 296, 3940, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 73, 4089, 0, 0, 8, 0, 0, 4070, 14, 2, 0, 0, 0, 15, 0, 0, 0, 4082, 0, 0, 0, 4096, 0, 0, 85, 4096, 0, 57, 0, 0, 0, 0, 0, 3779, 0] payload_rows_per_dst=[8192, 8008, 8164, 7638, 8162, 8166, 4736, 7776] dup_rows_per_dst=[0, 466, 2, 124, 296, 70, 112, 3624] combine_rows_per_src=[8192, 8008, 8164, 7638, 8162, 8166, 4736, 7776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63860 payload_rows_remote=55668 dup_rows=1676 dup_rows_remote=622 dedup_ratio=0.025574 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 3562, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 73, 4096, 0, 0, 0, 0, 0, 4086, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 527, 4095, 0, 71, 0, 0, 0, 0, 0, 3740, 0] payload_rows_per_dst=[8190, 7280, 8192, 8182, 8176, 8152, 8192, 7496] dup_rows_per_dst=[0, 348, 2, 146, 0, 0, 1054, 126] combine_rows_per_src=[8190, 7280, 8192, 8182, 8176, 8152, 8192, 7496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64366 payload_rows_remote=56180 dup_rows=1170 dup_rows_remote=1170 dedup_ratio=0.017853 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 84, 4096, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 142, 4095, 0, 47, 0, 0, 0, 0, 0, 3697, 0] payload_rows_per_dst=[8190, 7890, 8192, 8126, 8170, 8186, 8192, 7420] dup_rows_per_dst=[0, 658, 0, 160, 0, 0, 284, 68] combine_rows_per_src=[8190, 7890, 8192, 8126, 8170, 8186, 8192, 7420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=403572736 dispatch_meta_bytes=229672 combine_payload_bytes=403572736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403802408 ingress_bytes=513673216 link_bytes=513673216 token_bytes=807375144 weight_movement_bytes=110100480 total_nvlink_bytes=917475624 weight_to_token_ratio=0.136368 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.933622074e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64774 payload_rows_remote=56592 dup_rows=762 dup_rows_remote=752 dedup_ratio=0.011627 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 3873, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 61, 4095, 0, 0, 3, 0, 0, 4087, 5, 1, 0, 0, 0, 14, 0, 2, 0, 4087, 0, 0, 0, 4096, 0, 0, 128, 4096, 0, 61, 0, 0, 0, 0, 0, 3895, 0] payload_rows_per_dst=[8192, 7854, 8190, 8172, 8182, 8192, 8192, 7800] dup_rows_per_dst=[0, 252, 2, 116, 10, 14, 256, 112] combine_rows_per_src=[8192, 7854, 8190, 8172, 8182, 8192, 8192, 7800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64472 payload_rows_remote=56282 dup_rows=1064 dup_rows_remote=1046 dedup_ratio=0.016235 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 3930, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 50, 4096, 0, 0, 0, 0, 0, 4084, 1, 0, 1, 0, 0, 5, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 184, 4095, 0, 46, 0, 0, 0, 0, 0, 3886, 0] payload_rows_per_dst=[8190, 7910, 8192, 7854, 8170, 8190, 8176, 7790] dup_rows_per_dst=[18, 498, 14, 88, 2, 4, 366, 74] combine_rows_per_src=[8190, 7910, 8192, 7854, 8170, 8190, 8176, 7790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=380377088 dispatch_meta_bytes=216544 combine_payload_bytes=380377088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380593632 ingress_bytes=490477568 link_bytes=490477568 token_bytes=760970720 weight_movement_bytes=110100480 total_nvlink_bytes=871071200 weight_to_token_ratio=0.144684 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=30464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.530211604e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60436 payload_rows_remote=52334 dup_rows=5100 dup_rows_remote=4876 dedup_ratio=0.077820 router_tokens_per_expert=[0, 0, 0, 0, 0, 31, 0, 297, 0, 0, 3314, 0, 1580, 16, 0, 0, 0, 0, 110, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 116, 4096, 803, 0, 4048, 0, 0, 0, 0, 3122, 0, 0, 0, 0, 0, 0, 3271, 0, 0, 3818, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 38, 0, 0, 0, 0, 0] payload_rows_per_dst=[7686, 7464, 8102, 8192, 6244, 6542, 8014, 8192] dup_rows_per_dst=[606, 2356, 224, 1838, 0, 0, 0, 76] combine_rows_per_src=[7686, 7464, 8102, 8192, 6244, 6542, 8014, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=399028224 dispatch_meta_bytes=225160 combine_payload_bytes=399028224 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=399253384 ingress_bytes=839430144 link_bytes=839430144 token_bytes=798281608 weight_movement_bytes=440401920 total_nvlink_bytes=1238683528 weight_to_token_ratio=0.551687 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.459907083e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=402698240 dispatch_meta_bytes=229400 combine_payload_bytes=402698240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402927640 ingress_bytes=512798720 link_bytes=512798720 token_bytes=805625880 weight_movement_bytes=110100480 total_nvlink_bytes=915726360 weight_to_token_ratio=0.136665 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.918413150e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64454 payload_rows_remote=56346 dup_rows=1082 dup_rows_remote=1070 dedup_ratio=0.016510 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 3694, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 10, 4094, 0, 0, 5, 0, 0, 4084, 2, 0, 0, 0, 0, 11, 0, 5, 0, 4089, 0, 0, 0, 4096, 0, 0, 400, 4095, 0, 38, 0, 0, 0, 0, 0, 3913, 0] payload_rows_per_dst=[8190, 7578, 8192, 8108, 8178, 8190, 8192, 7826] dup_rows_per_dst=[0, 168, 2, 12, 4, 20, 800, 76] combine_rows_per_src=[8190, 7578, 8192, 8108, 8178, 8190, 8192, 7826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=405651456 dispatch_meta_bytes=229376 combine_payload_bytes=405651456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405880832 ingress_bytes=515751936 link_bytes=515751936 token_bytes=811532288 weight_movement_bytes=110100480 total_nvlink_bytes=921632768 weight_to_token_ratio=0.135670 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.969774433e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=403429376 dispatch_meta_bytes=229312 combine_payload_bytes=403429376 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=403658688 ingress_bytes=623630336 link_bytes=623630336 token_bytes=807088064 weight_movement_bytes=220200960 total_nvlink_bytes=1027289024 weight_to_token_ratio=0.272834 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.084595724e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=375130112 dispatch_meta_bytes=228840 combine_payload_bytes=375130112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375358952 ingress_bytes=485230592 link_bytes=485230592 token_bytes=750489064 weight_movement_bytes=110100480 total_nvlink_bytes=860589544 weight_to_token_ratio=0.146705 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.438958062e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=403888128 dispatch_meta_bytes=229664 combine_payload_bytes=403888128 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=404117792 ingress_bytes=954390528 link_bytes=954390528 token_bytes=808005920 weight_movement_bytes=550502400 total_nvlink_bytes=1358508320 weight_to_token_ratio=0.681310 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.659842098e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55520 payload_rows_remote=47560 dup_rows=10016 dup_rows_remote=7540 dedup_ratio=0.152832 router_tokens_per_expert=[3461, 0, 0, 3, 0, 499, 0, 0, 0, 2, 0, 4053, 3908, 0, 0, 0, 0, 0, 1, 0, 4, 5, 16, 0, 6, 2231, 4053, 1, 5, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 1321, 94, 1, 3, 0, 1, 0, 0, 0, 0, 1, 0, 4091, 0, 2, 742, 5, 0, 89, 2, 0, 0, 5, 4072, 0] payload_rows_per_dst=[7466, 7960, 8152, 8192, 3982, 8128, 3486, 8154] dup_rows_per_dst=[1224, 2476, 6, 4476, 838, 176, 638, 182] combine_rows_per_src=[7466, 7960, 8152, 8192, 3982, 8128, 3486, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60320 payload_rows_remote=52228 dup_rows=5216 dup_rows_remote=4998 dedup_ratio=0.079590 router_tokens_per_expert=[3765, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 4011, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 48, 9, 0, 0, 2184, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1582, 89, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 678, 0, 0, 48, 0, 0, 0, 0, 4044, 0] payload_rows_per_dst=[7620, 7726, 8096, 4368, 8192, 8130, 8096, 8092] dup_rows_per_dst=[64, 0, 278, 0, 3164, 158, 1334, 218] combine_rows_per_src=[7620, 7726, 8096, 4368, 8192, 8130, 8096, 8092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59506 payload_rows_remote=51314 dup_rows=6030 dup_rows_remote=3902 dedup_ratio=0.092010 router_tokens_per_expert=[2961, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 4069, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 2418, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1822, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1064, 0, 0, 18, 0, 0, 0, 0, 4068, 0] payload_rows_per_dst=[6066, 7758, 8156, 4836, 8192, 8166, 8192, 8140] dup_rows_per_dst=[70, 0, 4, 0, 3644, 152, 2128, 32] combine_rows_per_src=[6066, 7758, 8156, 4836, 8192, 8166, 8192, 8140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59724 payload_rows_remote=51538 dup_rows=5812 dup_rows_remote=5692 dedup_ratio=0.088684 router_tokens_per_expert=[3359, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 4048, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4, 0, 0, 2122, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1690, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1097, 0, 0, 18, 0, 0, 0, 0, 4001, 0] payload_rows_per_dst=[6892, 7888, 8124, 4244, 8192, 8186, 8192, 8006] dup_rows_per_dst=[80, 0, 6, 0, 3380, 120, 2194, 32] combine_rows_per_src=[6892, 7888, 8124, 4244, 8192, 8186, 8192, 8006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=340910080 dispatch_meta_bytes=220400 combine_payload_bytes=340910080 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=341130480 ingress_bytes=671211520 link_bytes=671211520 token_bytes=682040560 weight_movement_bytes=330301440 total_nvlink_bytes=1012342000 weight_to_token_ratio=0.484284 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.167347229e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=374370304 dispatch_meta_bytes=228904 combine_payload_bytes=374370304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374599208 ingress_bytes=484470784 link_bytes=484470784 token_bytes=748969512 weight_movement_bytes=110100480 total_nvlink_bytes=859069992 weight_to_token_ratio=0.147003 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.425743751e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60064 payload_rows_remote=51872 dup_rows=5472 dup_rows_remote=1928 dedup_ratio=0.083496 router_tokens_per_expert=[3386, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 4059, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 8, 11, 0, 0, 2094, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1772, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 692, 0, 0, 21, 0, 0, 0, 0, 4049, 0] payload_rows_per_dst=[7118, 7948, 8148, 4188, 8192, 8180, 8192, 8098] dup_rows_per_dst=[334, 0, 8, 0, 3544, 160, 1384, 42] combine_rows_per_src=[7118, 7948, 8148, 4188, 8192, 8180, 8192, 8098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53844 payload_rows_remote=45652 dup_rows=11692 dup_rows_remote=5042 dedup_ratio=0.178406 router_tokens_per_expert=[2940, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 4050, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 7, 31, 0, 0, 1926, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1983, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1024, 0, 0, 32, 0, 0, 0, 0, 4080, 0] payload_rows_per_dst=[8192, 7914, 8164, 8192, 3966, 7208, 2048, 8160] dup_rows_per_dst=[6650, 0, 12, 4826, 0, 140, 0, 64] combine_rows_per_src=[8192, 7914, 8164, 8192, 3966, 7208, 2048, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=369424384 dispatch_meta_bytes=228920 combine_payload_bytes=369424384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369653304 ingress_bytes=479524864 link_bytes=479524864 token_bytes=739077688 weight_movement_bytes=110100480 total_nvlink_bytes=849178168 weight_to_token_ratio=0.148970 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.339726068e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64814 payload_rows_remote=56626 dup_rows=722 dup_rows_remote=722 dedup_ratio=0.011017 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 18, 4094, 0, 0, 0, 0, 0, 4086, 0, 0, 1, 0, 0, 4, 0, 1, 0, 4083, 0, 0, 0, 4096, 0, 0, 69, 4095, 0, 85, 0, 0, 0, 0, 0, 3791, 0] payload_rows_per_dst=[8190, 8072, 8188, 8182, 8172, 8174, 8192, 7644] dup_rows_per_dst=[0, 436, 0, 36, 2, 2, 138, 108] combine_rows_per_src=[8190, 8072, 8188, 8182, 8172, 8174, 8192, 7644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=367818752 dispatch_meta_bytes=220864 combine_payload_bytes=367818752 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=368039616 ingress_bytes=808220672 link_bytes=808220672 token_bytes=735858368 weight_movement_bytes=440401920 total_nvlink_bytes=1176260288 weight_to_token_ratio=0.598487 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=25856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.405628678e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6755396096 prefetch_bytes=1211105280 reduce_bytes=880803840 weight_movement_bytes=2091909120 egress_bytes=3378809344 ingress_bytes=5468495872 link_bytes=5468495872 total_nvlink_bytes=8847305216 weight_to_token_ratio=0.309665 prefetch_share_of_ingress=0.221470 dedup_ratio=0.164795 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=8 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044853 gemm_inner_pad_upper_bound=8255 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.510613734e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58590 payload_rows_remote=52048 dup_rows=6946 dup_rows_remote=4038 dedup_ratio=0.105988 router_tokens_per_expert=[2914, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 4052, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 10, 11, 0, 0, 2541, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1865, 68, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 833, 0, 0, 7, 0, 0, 0, 0, 4086, 0] payload_rows_per_dst=[6056, 7950, 8134, 6542, 5368, 8176, 8192, 8172] dup_rows_per_dst=[30, 0, 12, 2908, 2186, 124, 1672, 14] combine_rows_per_src=[6056, 7950, 8134, 6542, 5368, 8176, 8192, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=371818496 dispatch_meta_bytes=215200 combine_payload_bytes=371818496 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=372033696 ingress_bytes=592019456 link_bytes=592019456 token_bytes=743852192 weight_movement_bytes=220200960 total_nvlink_bytes=964053152 weight_to_token_ratio=0.296028 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.029619204e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6250751512 prefetch_bytes=770703360 reduce_bytes=1431306240 weight_movement_bytes=2202009600 egress_bytes=3126464024 ingress_bytes=5326297088 link_bytes=5326297088 total_nvlink_bytes=8452761112 weight_to_token_ratio=0.352279 prefetch_share_of_ingress=0.144698 dedup_ratio=0.215909 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=13 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044853 gemm_inner_pad_upper_bound=8382 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.263306661e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=327233536 dispatch_meta_bytes=202776 combine_payload_bytes=327233536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=327436312 ingress_bytes=437334016 link_bytes=437334016 token_bytes=654669848 weight_movement_bytes=110100480 total_nvlink_bytes=764770328 weight_to_token_ratio=0.168177 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=26624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.605957829e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6954346552 prefetch_bytes=1101004800 reduce_bytes=440401920 weight_movement_bytes=1541406720 egress_bytes=3478325304 ingress_bytes=5017427968 link_bytes=5017427968 total_nvlink_bytes=8495753272 weight_to_token_ratio=0.221647 prefetch_share_of_ingress=0.219436 dedup_ratio=0.154559 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=4 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=9017 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.726132461e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6759525016 prefetch_bytes=880803840 reduce_bytes=1871708160 weight_movement_bytes=2752512000 egress_bytes=3380902552 ingress_bytes=6131134464 link_bytes=6131134464 total_nvlink_bytes=9512037016 weight_to_token_ratio=0.407205 prefetch_share_of_ingress=0.143661 dedup_ratio=0.176358 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=17 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044853 gemm_inner_pad_upper_bound=8509 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.066305123e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=405895168 dispatch_meta_bytes=229392 combine_payload_bytes=405895168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406124560 ingress_bytes=515995648 link_bytes=515995648 token_bytes=812019728 weight_movement_bytes=110100480 total_nvlink_bytes=922120208 weight_to_token_ratio=0.135588 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.974012986e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7119163448 prefetch_bytes=880803840 reduce_bytes=1321205760 weight_movement_bytes=2202009600 egress_bytes=3560724536 ingress_bytes=5760448512 link_bytes=5760448512 total_nvlink_bytes=9321173048 weight_to_token_ratio=0.309307 prefetch_share_of_ingress=0.152905 dedup_ratio=0.134967 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=12 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044485 gemm_inner_pad_upper_bound=8509 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.001836739e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=373080064 dispatch_meta_bytes=224344 combine_payload_bytes=373080064 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=373304408 ingress_bytes=593281024 link_bytes=593281024 token_bytes=746384472 weight_movement_bytes=220200960 total_nvlink_bytes=966585432 weight_to_token_ratio=0.295024 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=17920 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.031813279e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6336279056 prefetch_bytes=1321205760 reduce_bytes=440401920 weight_movement_bytes=1761607680 egress_bytes=3169212944 ingress_bytes=4928673792 link_bytes=4928673792 total_nvlink_bytes=8097886736 weight_to_token_ratio=0.278019 prefetch_share_of_ingress=0.268065 dedup_ratio=0.206342 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=4 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.045221 gemm_inner_pad_upper_bound=8382 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.571774352e-05 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7103519312 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=3552907856 ingress_bytes=5752621056 link_bytes=5752621056 total_nvlink_bytes=9305528912 weight_to_token_ratio=0.309989 prefetch_share_of_ingress=0.172253 dedup_ratio=0.139514 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044118 gemm_inner_pad_upper_bound=8763 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.000475416e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58944 payload_rows_remote=50770 dup_rows=6592 dup_rows_remote=6530 dedup_ratio=0.100586 router_tokens_per_expert=[3749, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 4051, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 57, 10, 0, 0, 2286, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1348, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 798, 0, 0, 58, 0, 0, 0, 0, 4007, 0] payload_rows_per_dst=[7586, 7916, 8174, 8192, 2696, 8174, 8192, 8014] dup_rows_per_dst=[88, 0, 62, 4572, 0, 158, 1596, 116] combine_rows_per_src=[7586, 7916, 8174, 8192, 2696, 8174, 8192, 8014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=363919360 dispatch_meta_bytes=229200 combine_payload_bytes=363919360 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=364148560 ingress_bytes=584120320 link_bytes=584120320 token_bytes=728067920 weight_movement_bytes=220200960 total_nvlink_bytes=948268880 weight_to_token_ratio=0.302446 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.015881308e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7011089936 prefetch_bytes=1211105280 reduce_bytes=770703360 weight_movement_bytes=1981808640 egress_bytes=3506697744 ingress_bytes=5486200832 link_bytes=5486200832 total_nvlink_bytes=8992898576 weight_to_token_ratio=0.282668 prefetch_share_of_ingress=0.220755 dedup_ratio=0.143817 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=7 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=9144 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.541405572e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none + [2026-08-26 18:09:55] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 368.5 | throughput per GPU (TFLOP/s/GPU): 335.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.469075E+00 | loss scale: 1.0 | grad norm: 4.150 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 18:09:55 +[rank0]:[W826 18:09:56.685429842 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 18:09:57.680901173 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 18:09:57.681244782 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 18:09:57.697097115 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 18:09:57.769063897 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 18:09:58.275342726 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 18:09:58.410084021 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 18:09:58.410286669 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 18:09:59.447000 462358 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 18:09:59.447000 462358 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 462451 closing signal SIGTERM +W0826 18:09:59.448000 462358 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 462452 closing signal SIGTERM +W0826 18:09:59.450000 462358 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 462453 closing signal SIGTERM +W0826 18:09:59.451000 462358 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 462454 closing signal SIGTERM +W0826 18:09:59.455000 462358 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 462455 closing signal SIGTERM +W0826 18:09:59.456000 462358 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 462456 closing signal SIGTERM +W0826 18:09:59.459000 462358 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 462457 closing signal SIGTERM +W0826 18:09:59.461000 462358 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 462458 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 462358 got signal: 15 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/mem_samples.csv b/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/mem_samples.csv index fa9efe04059d1874d9e5ca4b36cc83a4543bf908..ea001bd9c1b5429fce5bcc3f24450660698f7b16 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/mem_samples.csv +++ b/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/mem_samples.csv @@ -39,21 +39,21 @@ 6, 4 7, 4 0, 4 -1, 47 -2, 629 -3, 629 -4, 629 -5, 47 -6, 629 -7, 629 -0, 927 -1, 903 -2, 913 -3, 903 -4, 907 -5, 913 -6, 929 -7, 919 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 0, 1803 1, 1803 2, 1803 @@ -62,30 +62,30 @@ 5, 1803 6, 1803 7, 1803 -0, 6769 -1, 6769 -2, 6763 -3, 6779 -4, 6785 -5, 6771 -6, 6749 -7, 6769 -0, 58457 -1, 57867 -2, 57895 -3, 58575 -4, 58571 -5, 58555 -6, 58549 -7, 58621 -0, 59439 -1, 58873 -2, 58871 -3, 59571 -4, 59591 -5, 59585 -6, 59581 -7, 59591 +0, 6739 +1, 6731 +2, 6755 +3, 6765 +4, 6747 +5, 6737 +6, 6751 +7, 6763 +0, 58421 +1, 57855 +2, 57909 +3, 58607 +4, 58529 +5, 58551 +6, 58565 +7, 58547 +0, 59091 +1, 58523 +2, 58523 +3, 59223 +4, 59223 +5, 59223 +6, 59223 +7, 59223 0, 59719 1, 59151 2, 59151 @@ -94,30 +94,30 @@ 5, 59851 6, 59851 7, 59851 -0, 60421 -1, 59851 -2, 59851 -3, 60551 -4, 60551 -5, 60551 -6, 60551 -7, 60551 -0, 89393 -1, 88729 -2, 84787 -3, 86505 -4, 89453 -5, 86055 -6, 86483 -7, 86565 -0, 118011 -1, 118087 -2, 118059 -3, 118071 -4, 118061 -5, 118093 -6, 118087 -7, 118101 +0, 59723 +1, 59153 +2, 59153 +3, 59853 +4, 59853 +5, 59853 +6, 59853 +7, 59853 +0, 90309 +1, 92135 +2, 83293 +3, 83049 +4, 81849 +5, 81853 +6, 82825 +7, 83235 +0, 118125 +1, 118181 +2, 118167 +3, 118201 +4, 118195 +5, 118167 +6, 118219 +7, 118183 0, 118939 1, 118985 2, 118967 @@ -142,14 +142,14 @@ 5, 119663 6, 119663 7, 119663 -0, 128675 -1, 128377 -2, 128485 -3, 128489 -4, 128499 -5, 128543 -6, 128461 -7, 128469 +0, 128673 +1, 128335 +2, 128499 +3, 128471 +4, 128461 +5, 128523 +6, 128471 +7, 128515 0, 141327 1, 141555 2, 141531 @@ -174,19 +174,19 @@ 5, 140711 6, 140711 7, 140711 -0, 138373 -1, 138989 -2, 139353 -3, 138791 +0, 138811 +1, 138687 +2, 138977 +3, 139001 4, 138403 -5, 138791 -6, 138403 -7, 138791 -0, 12601 +5, 138403 +6, 138791 +7, 138985 +0, 13243 1, 12604 -2, 12732 -3, 138789 -4, 13244 +2, 12604 +3, 12604 +4, 12604 5, 12604 -6, 12604 -7, 12604 +6, 12732 +7, 138789 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/train.log b/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/train.log index 5c8af7bc127e62bf6c900858823d2665caf26e74..8f1d2441688ddf0be9f44741febd95a73d4a9dc3 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/train.log +++ b/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/train.log @@ -1,5 +1,5 @@ -[16:57:59] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[16:57:59] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads +[18:10:05] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[18:10:05] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads ============================================================== QWEN2_57B / moonep (slots/rank=8 policy=matched) E=64 K=8 H'=2560 m=8 @@ -8,17 +8,10 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ 35ae6e7f ============================================================== -W0826 16:58:01.329000 292582 torch/distributed/run.py:766] -W0826 16:58:01.329000 292582 torch/distributed/run.py:766] ***************************************** -W0826 16:58:01.329000 292582 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 16:58:01.329000 292582 torch/distributed/run.py:766] ***************************************** -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +W0826 18:10:07.040000 465016 torch/distributed/run.py:766] +W0826 18:10:07.040000 465016 torch/distributed/run.py:766] ***************************************** +W0826 18:10:07.040000 465016 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 18:10:07.040000 465016 torch/distributed/run.py:766] ***************************************** /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled @@ -573,6 +566,10 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... @@ -580,22 +577,25 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 7 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : -7 -[Gloo] Rank [Gloo] Rank 04 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 01 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to [Gloo] Rank 7[Gloo] Rank 2 peer ranks. 3 is connected to Expected number of connected peer ranks is : is connected to 777 peer ranks. - peer ranks. Expected number of connected peer ranks is : [Gloo] Rank Expected number of connected peer ranks is : 757 - is connected to -7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 57 is connected to peer ranks. [Gloo] Rank 7Expected number of connected peer ranks is : 7 peer ranks. 7 is connected to Expected number of connected peer ranks is : 7 +7 peer ranks. +Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 @@ -607,24 +607,24 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank4]:[W826 16:58:19.727931468 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 16:58:19.727946994 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 16:58:19.727975738 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 16:58:19.727984827 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 16:58:19.728011091 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 16:58:19.728097199 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 16:58:19.728110832 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 18:10:25.266221442 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 18:10:25.267819905 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 18:10:25.267944559 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 18:10:25.267943088 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 18:10:25.267988931 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 18:10:25.269164584 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 18:10:25.273034248 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.034 seconds +>>> done with dataset index builder. Compilation time: 0.033 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 16:58:19.873974344 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.753 seconds -time to initialize megatron (seconds): 7.336 -[after megatron is initialized] datetime: 2026-08-26 16:58:25 +[rank0]:[W826 18:10:25.417250529 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.806 seconds +time to initialize megatron (seconds): 7.348 +[after megatron is initialized] datetime: 2026-08-26 18:10:31 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -655,8 +655,8 @@ Params for bucket 3 (146800640 elements, 146800640 padded size): Params for bucket 4 (103001600 elements, 103001600 padded size): module.decoder.layers.9.mlp.router.weight module.decoder.layers.9.self_attention.linear_qkv.bias - module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight @@ -664,22 +664,22 @@ Params for bucket 5 (146800640 elements, 146800640 padded size): module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight Params for bucket 6 (103001600 elements, 103001600 padded size): module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.bias + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.8.self_attention.linear_proj.weight module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.8.self_attention.linear_qkv.bias - module.decoder.layers.8.self_attention.linear_qkv.weight Params for bucket 7 (146800640 elements, 146800640 padded size): module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight Params for bucket 8 (103001600 elements, 103001600 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.bias module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.7.self_attention.linear_qkv.weight module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.self_attention.linear_proj.weight module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.self_attention.linear_qkv.bias Params for bucket 9 (146800640 elements, 146800640 padded size): module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight Params for bucket 10 (103001600 elements, 103001600 padded size): @@ -688,46 +688,46 @@ Params for bucket 10 (103001600 elements, 103001600 padded size): module.decoder.layers.6.pre_mlp_layernorm.weight module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.6.self_attention.linear_qkv.bias module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.bias Params for bucket 11 (146800640 elements, 146800640 padded size): module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight Params for bucket 12 (103001600 elements, 103001600 padded size): - module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.5.pre_mlp_layernorm.weight - module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.5.self_attention.linear_qkv.bias + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight Params for bucket 13 (146800640 elements, 146800640 padded size): module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight Params for bucket 14 (103001600 elements, 103001600 padded size): - module.decoder.layers.4.self_attention.linear_qkv.weight - module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.self_attention.linear_proj.weight - module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.4.self_attention.linear_qkv.bias + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_proj.weight Params for bucket 15 (146800640 elements, 146800640 padded size): module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight Params for bucket 16 (103001600 elements, 103001600 padded size): module.decoder.layers.3.self_attention.linear_qkv.bias - module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.self_attention.linear_qkv.weight - module.decoder.layers.3.self_attention.linear_proj.weight - module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight Params for bucket 17 (146800640 elements, 146800640 padded size): module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight Params for bucket 18 (103001600 elements, 103001600 padded size): - module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.2.self_attention.linear_qkv.bias module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.mlp.router.weight module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight Params for bucket 19 (146800640 elements, 146800640 padded size): @@ -737,26 +737,26 @@ Params for bucket 20 (103001600 elements, 103001600 padded size): module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.1.self_attention.linear_qkv.bias module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.1.self_attention.linear_qkv.weight - module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.weight Params for bucket 21 (146800640 elements, 146800640 padded size): module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight Params for bucket 22 (209890816 elements, 209890816 padded size): - module.decoder.layers.0.pre_mlp_layernorm.weight - module.decoder.layers.0.self_attention.linear_proj.weight module.decoder.layers.0.mlp.router.weight - module.decoder.layers.0.self_attention.linear_qkv.bias - module.decoder.layers.0.self_attention.linear_qkv.weight module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_qkv.bias + module.decoder.layers.0.self_attention.linear_proj.weight module.embedding.word_embeddings.weight + module.decoder.layers.0.pre_mlp_layernorm.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 Params for bucket 1 (45875200 elements, 45875200 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 Params for bucket 2 (45875200 elements, 45875200 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 @@ -776,8 +776,8 @@ Params for bucket 5 (45875200 elements, 45875200 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight6 Params for bucket 6 (45875200 elements, 45875200 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.8.mlp.experts.linear_fc2.weight4 module.decoder.layers.8.mlp.experts.linear_fc2.weight1 module.decoder.layers.8.mlp.experts.linear_fc2.weight2 @@ -794,30 +794,30 @@ Params for bucket 9 (45875200 elements, 45875200 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc1.weight0 Params for bucket 10 (45875200 elements, 45875200 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 module.decoder.layers.7.mlp.experts.linear_fc2.weight6 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 - module.decoder.layers.7.mlp.experts.linear_fc2.weight5 - module.decoder.layers.7.mlp.experts.linear_fc2.weight3 Params for bucket 11 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight1 - module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 Params for bucket 12 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight5 - module.decoder.layers.7.mlp.experts.linear_fc1.weight3 module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 Params for bucket 13 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight2 - module.decoder.layers.7.mlp.experts.linear_fc1.weight1 module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 Params for bucket 14 (45875200 elements, 45875200 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 Params for bucket 15 (45875200 elements, 45875200 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight2 module.decoder.layers.6.mlp.experts.linear_fc2.weight1 @@ -837,48 +837,48 @@ Params for bucket 18 (45875200 elements, 45875200 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight6 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 Params for bucket 19 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight0 - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 Params for bucket 20 (55050240 elements, 55050240 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight5 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 Params for bucket 21 (55050240 elements, 55050240 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight4 - module.decoder.layers.5.mlp.experts.linear_fc1.weight3 module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 Params for bucket 22 (45875200 elements, 45875200 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight1 module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 Params for bucket 23 (45875200 elements, 45875200 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight3 - module.decoder.layers.4.mlp.experts.linear_fc2.weight2 - module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 Params for bucket 24 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc1.weight6 module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight0 Params for bucket 25 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight3 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 Params for bucket 26 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight2 module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.4.mlp.experts.linear_fc1.weight0 Params for bucket 27 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 - module.decoder.layers.3.mlp.experts.linear_fc2.weight6 - module.decoder.layers.3.mlp.experts.linear_fc2.weight4 Params for bucket 28 (45875200 elements, 45875200 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight7 @@ -890,37 +890,37 @@ Params for bucket 29 (55050240 elements, 55050240 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight5 Params for bucket 30 (55050240 elements, 55050240 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight3 - module.decoder.layers.3.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 Params for bucket 31 (45875200 elements, 45875200 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight7 module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 Params for bucket 32 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 module.decoder.layers.2.mlp.experts.linear_fc2.weight3 - module.decoder.layers.2.mlp.experts.linear_fc2.weight0 - module.decoder.layers.2.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc2.weight2 - module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 Params for bucket 33 (55050240 elements, 55050240 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight7 - module.decoder.layers.2.mlp.experts.linear_fc1.weight5 module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 Params for bucket 34 (55050240 elements, 55050240 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight4 module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.2.mlp.experts.linear_fc1.weight3 Params for bucket 35 (45875200 elements, 45875200 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight1 - module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 Params for bucket 36 (45875200 elements, 45875200 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight5 module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 - module.decoder.layers.1.mlp.experts.linear_fc2.weight4 module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 Params for bucket 37 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight1 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 @@ -942,22 +942,22 @@ Params for bucket 40 (45875200 elements, 45875200 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight5 Params for bucket 41 (45875200 elements, 45875200 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 module.decoder.layers.0.mlp.experts.linear_fc1.weight7 module.decoder.layers.0.mlp.experts.linear_fc2.weight1 - module.decoder.layers.0.mlp.experts.linear_fc2.weight0 Params for bucket 42 (55050240 elements, 55050240 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight6 - module.decoder.layers.0.mlp.experts.linear_fc1.weight4 module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 Params for bucket 43 (55050240 elements, 55050240 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight3 - module.decoder.layers.0.mlp.experts.linear_fc1.weight2 module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 Params for bucket 44 (18350080 elements, 18350080 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:58:28 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:10:33 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -965,7 +965,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -978,14 +978,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42 INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 16:58:28 +[after dataloaders are built] datetime: 2026-08-26 18:10:33 done with setup ... training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (2551.39, 2557.31) - train/valid/test-data-iterators-setup ..........: (57.08, 77.26) -[before the start of training step] datetime: 2026-08-26 16:58:28 + model-and-optimizer-setup ......................: (2454.10, 2460.37) + train/valid/test-data-iterators-setup ..........: (56.58, 65.91) +[before the start of training step] datetime: 2026-08-26 18:10:33 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=4096, H=3584, E=64, K=8, NvS=34800, NvS_padded=34816 sm_count : num_sms=32, num_sms_dedup=148 @@ -994,2522 +994,2522 @@ INFO:moonep.api:MoonEP comm buffer sizes: local_temp_buffer_size : 1.26 MiB total_size : 241.26 MiB WARNING:megatron.core.rerun_state_machine:Result validation enabled -Number of parameters in transformer block in billions: 20.11 [2026-08-26 16:58:51] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22975.5 | throughput per GPU (TFLOP/s/GPU): 5.4 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097734E+01 | loss scale: 1.0 | grad norm: 29.722 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:10:56] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22574.1 | throughput per GPU (TFLOP/s/GPU): 5.5 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097734E+01 | loss scale: 1.0 | grad norm: 29.717 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 20.11 Number of parameters in embedding layers in billions: 0.36 Total number of parameters in billions: 20.47 Number of parameters in most loaded shard in billions: 20.4724 Theoretical memory footprints: weight and optimizer=146430.14 MB [Rank 0] (after 1 iterations) memory (MB) | allocated: 104985.1162109375 | max allocated: 104985.11669921875 | reserved: 108632.0 | max reserved: 108632.0 - [2026-08-26 16:58:51] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 374.7 | throughput per GPU (TFLOP/s/GPU): 329.6 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097746E+01 | loss scale: 1.0 | grad norm: 30.463 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44592 payload_rows_remote=39054 dup_rows=20944 dup_rows_remote=18276 dedup_ratio=0.319580 router_tokens_per_expert=[493, 476, 508, 502, 505, 493, 489, 525, 491, 482, 513, 527, 514, 519, 517, 519, 503, 564, 525, 520, 529, 483, 530, 505, 476, 518, 517, 521, 493, 519, 525, 540, 514, 510, 526, 499, 505, 523, 515, 492, 510, 525, 530, 521, 547, 513, 525, 513, 531, 482, 527, 513, 502, 509, 505, 487, 511, 484, 497, 524, 513, 527, 541, 506] payload_rows_per_dst=[5496, 5592, 5616, 5608, 5580, 5376, 5786, 5538] dup_rows_per_dst=[2486, 2572, 2658, 2610, 2632, 2406, 2912, 2668] combine_rows_per_src=[5496, 5592, 5616, 5608, 5580, 5376, 5786, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38988 dup_rows=20976 dup_rows_remote=18408 dedup_ratio=0.320068 router_tokens_per_expert=[460, 523, 495, 549, 527, 473, 502, 488, 493, 518, 510, 491, 551, 499, 489, 519, 510, 501, 526, 505, 539, 470, 479, 521, 509, 514, 484, 535, 511, 507, 522, 591, 496, 502, 493, 487, 550, 510, 531, 501, 540, 514, 472, 562, 520, 543, 487, 517, 461, 494, 475, 526, 519, 519, 558, 528, 572, 496, 481, 500, 523, 519, 530, 531] payload_rows_per_dst=[5480, 5572, 5498, 5620, 5546, 5652, 5574, 5618] dup_rows_per_dst=[2554, 2568, 2604, 2726, 2594, 2658, 2586, 2686] combine_rows_per_src=[5480, 5572, 5498, 5620, 5546, 5652, 5574, 5618] + [2026-08-26 18:10:56] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 416.3 | throughput per GPU (TFLOP/s/GPU): 296.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097746E+01 | loss scale: 1.0 | grad norm: 30.465 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44298 payload_rows_remote=38800 dup_rows=21238 dup_rows_remote=18596 dedup_ratio=0.324066 router_tokens_per_expert=[483, 512, 499, 513, 538, 496, 559, 520, 502, 521, 501, 531, 508, 522, 537, 508, 511, 502, 498, 541, 529, 528, 499, 502, 512, 536, 501, 515, 524, 521, 517, 469, 476, 562, 498, 496, 494, 497, 497, 550, 492, 527, 493, 522, 528, 514, 526, 491, 503, 483, 531, 507, 476, 496, 524, 501, 523, 515, 557, 511, 484, 521, 498, 520] payload_rows_per_dst=[5602, 5486, 5532, 5558, 5498, 5538, 5458, 5626] dup_rows_per_dst=[2638, 2774, 2688, 2632, 2642, 2648, 2584, 2632] combine_rows_per_src=[5602, 5486, 5532, 5558, 5498, 5538, 5458, 5626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38842 dup_rows=21086 dup_rows_remote=18480 dedup_ratio=0.321747 router_tokens_per_expert=[503, 544, 506, 539, 511, 552, 514, 511, 487, 496, 517, 508, 525, 510, 505, 496, 516, 482, 506, 482, 498, 509, 523, 523, 485, 490, 505, 476, 509, 511, 531, 523, 508, 531, 550, 518, 541, 523, 549, 539, 518, 496, 490, 484, 502, 536, 546, 535, 513, 523, 494, 537, 498, 477, 481, 497, 525, 544, 498, 504, 494, 525, 480, 519] payload_rows_per_dst=[5598, 5454, 5492, 5574, 5706, 5608, 5522, 5496] dup_rows_per_dst=[2762, 2634, 2586, 2486, 2812, 2606, 2518, 2682] combine_rows_per_src=[5598, 5454, 5492, 5574, 5706, 5608, 5522, 5496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38700 dup_rows=21092 dup_rows_remote=18192 dedup_ratio=0.321838 router_tokens_per_expert=[524, 456, 538, 530, 533, 553, 486, 544, 540, 490, 540, 509, 519, 506, 548, 498, 492, 528, 528, 503, 508, 496, 539, 511, 517, 504, 505, 522, 513, 552, 511, 488, 486, 477, 516, 512, 518, 501, 482, 502, 535, 488, 505, 503, 521, 533, 502, 557, 491, 481, 540, 488, 494, 509, 463, 553, 471, 502, 486, 509, 515, 523, 538, 536] payload_rows_per_dst=[5744, 5634, 5538, 5626, 5490, 5538, 5506, 5368] dup_rows_per_dst=[2900, 2800, 2674, 2756, 2498, 2630, 2532, 2302] combine_rows_per_src=[5744, 5634, 5538, 5626, 5490, 5538, 5506, 5368] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44496 payload_rows_remote=38914 dup_rows=21040 dup_rows_remote=18484 dedup_ratio=0.321045 router_tokens_per_expert=[510, 516, 504, 545, 510, 487, 524, 513, 546, 510, 506, 512, 503, 490, 491, 521, 501, 513, 487, 493, 543, 519, 524, 491, 519, 520, 516, 493, 484, 510, 513, 494, 517, 531, 560, 504, 486, 510, 503, 493, 476, 545, 523, 509, 528, 519, 531, 505, 515, 527, 520, 513, 485, 523, 506, 480, 543, 513, 539, 539, 514, 484, 517, 502] payload_rows_per_dst=[5574, 5558, 5508, 5498, 5538, 5584, 5582, 5654] dup_rows_per_dst=[2644, 2600, 2634, 2600, 2670, 2688, 2556, 2648] combine_rows_per_src=[5574, 5558, 5508, 5498, 5538, 5584, 5582, 5654] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44520 payload_rows_remote=39116 dup_rows=21016 dup_rows_remote=18448 dedup_ratio=0.320679 router_tokens_per_expert=[487, 493, 509, 515, 497, 507, 511, 517, 522, 524, 462, 509, 498, 492, 512, 500, 503, 504, 485, 518, 493, 525, 561, 506, 481, 477, 467, 574, 502, 465, 519, 501, 481, 508, 504, 493, 557, 509, 525, 526, 498, 512, 488, 503, 549, 542, 544, 506, 533, 531, 515, 499, 553, 534, 496, 514, 463, 553, 498, 527, 563, 531, 547, 530] payload_rows_per_dst=[5506, 5538, 5586, 5404, 5568, 5570, 5678, 5670] dup_rows_per_dst=[2566, 2500, 2604, 2568, 2638, 2714, 2672, 2754] combine_rows_per_src=[5506, 5538, 5586, 5404, 5568, 5570, 5678, 5670] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38700 dup_rows=21092 dup_rows_remote=18192 dedup_ratio=0.321838 router_tokens_per_expert=[524, 456, 538, 530, 533, 553, 486, 544, 540, 490, 540, 509, 519, 506, 548, 498, 492, 528, 528, 503, 508, 496, 539, 511, 517, 504, 505, 522, 513, 552, 511, 488, 486, 477, 516, 512, 518, 501, 482, 502, 535, 488, 505, 503, 521, 533, 502, 557, 491, 481, 540, 488, 494, 509, 463, 553, 471, 502, 486, 509, 515, 523, 538, 536] payload_rows_per_dst=[5744, 5634, 5538, 5626, 5490, 5538, 5506, 5368] dup_rows_per_dst=[2900, 2800, 2674, 2756, 2498, 2630, 2532, 2302] combine_rows_per_src=[5744, 5634, 5538, 5626, 5490, 5538, 5506, 5368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38842 dup_rows=21086 dup_rows_remote=18480 dedup_ratio=0.321747 router_tokens_per_expert=[503, 544, 506, 539, 511, 552, 514, 511, 487, 496, 517, 508, 525, 510, 505, 496, 516, 482, 506, 482, 498, 509, 523, 523, 485, 490, 505, 476, 509, 511, 531, 523, 508, 531, 550, 518, 541, 523, 549, 539, 518, 496, 490, 484, 502, 536, 546, 535, 513, 523, 494, 537, 498, 477, 481, 497, 525, 544, 498, 504, 494, 525, 480, 519] payload_rows_per_dst=[5598, 5454, 5492, 5574, 5706, 5608, 5522, 5496] dup_rows_per_dst=[2762, 2634, 2586, 2486, 2812, 2606, 2518, 2682] combine_rows_per_src=[5598, 5454, 5492, 5574, 5706, 5608, 5522, 5496] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=38964 dup_rows=20948 dup_rows_remote=18254 dedup_ratio=0.319641 router_tokens_per_expert=[498, 478, 496, 493, 542, 512, 510, 464, 509, 513, 517, 542, 454, 553, 527, 512, 522, 507, 508, 549, 493, 522, 545, 513, 498, 544, 473, 517, 527, 539, 509, 528, 517, 519, 520, 477, 488, 523, 522, 496, 536, 494, 501, 523, 535, 501, 517, 553, 506, 507, 544, 492, 504, 490, 514, 478, 491, 513, 513, 479, 549, 528, 517, 507] payload_rows_per_dst=[5448, 5540, 5624, 5598, 5656, 5662, 5480, 5580] dup_rows_per_dst=[2538, 2714, 2694, 2672, 2468, 2658, 2590, 2614] combine_rows_per_src=[5448, 5540, 5624, 5598, 5656, 5662, 5480, 5580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279939072 dispatch_meta_bytes=229320 combine_payload_bytes=279939072 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=280168392 ingress_bytes=720340992 link_bytes=720340992 token_bytes=560107464 weight_movement_bytes=440401920 total_nvlink_bytes=1000509384 weight_to_token_ratio=0.786281 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252791461e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=229584 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279695568 ingress_bytes=389566464 link_bytes=389566464 token_bytes=559161552 weight_movement_bytes=110100480 total_nvlink_bytes=669262032 weight_to_token_ratio=0.196903 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775201536e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278118400 dispatch_meta_bytes=229584 combine_payload_bytes=278118400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278347984 ingress_bytes=388218880 link_bytes=388218880 token_bytes=556466384 weight_movement_bytes=110100480 total_nvlink_bytes=666566864 weight_to_token_ratio=0.197856 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751764834e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277401600 dispatch_meta_bytes=227568 combine_payload_bytes=277401600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277629168 ingress_bytes=387502080 link_bytes=387502080 token_bytes=555030768 weight_movement_bytes=110100480 total_nvlink_bytes=665131248 weight_to_token_ratio=0.198368 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.739298503e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=229288 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278648744 ingress_bytes=608720896 link_bytes=608720896 token_bytes=557068200 weight_movement_bytes=330301440 total_nvlink_bytes=887369640 weight_to_token_ratio=0.592928 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058665756e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=229592 combine_payload_bytes=278935552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279165144 ingress_bytes=389036032 link_bytes=389036032 token_bytes=558100696 weight_movement_bytes=110100480 total_nvlink_bytes=668201176 weight_to_token_ratio=0.197277 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765976451e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44592 payload_rows_remote=39054 dup_rows=20944 dup_rows_remote=18276 dedup_ratio=0.319580 router_tokens_per_expert=[493, 476, 508, 502, 505, 493, 489, 525, 491, 482, 513, 527, 514, 519, 517, 519, 503, 564, 525, 520, 529, 483, 530, 505, 476, 518, 517, 521, 493, 519, 525, 540, 514, 510, 526, 499, 505, 523, 515, 492, 510, 525, 530, 521, 547, 513, 525, 513, 531, 482, 527, 513, 502, 509, 505, 487, 511, 484, 497, 524, 513, 527, 541, 506] payload_rows_per_dst=[5496, 5592, 5616, 5608, 5580, 5376, 5786, 5538] dup_rows_per_dst=[2486, 2572, 2658, 2610, 2632, 2406, 2912, 2668] combine_rows_per_src=[5496, 5592, 5616, 5608, 5580, 5376, 5786, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277401600 dispatch_meta_bytes=227568 combine_payload_bytes=277401600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277629168 ingress_bytes=387502080 link_bytes=387502080 token_bytes=555030768 weight_movement_bytes=110100480 total_nvlink_bytes=665131248 weight_to_token_ratio=0.198368 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.739298503e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280383488 dispatch_meta_bytes=230256 combine_payload_bytes=280383488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280613744 ingress_bytes=390483968 link_bytes=390483968 token_bytes=560997232 weight_movement_bytes=110100480 total_nvlink_bytes=671097712 weight_to_token_ratio=0.196259 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.791158439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=228872 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279522824 ingress_bytes=499494912 link_bytes=499494912 token_bytes=558816776 weight_movement_bytes=220200960 total_nvlink_bytes=779017736 weight_to_token_ratio=0.394049 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687038048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38962 dup_rows=20926 dup_rows_remote=18250 dedup_ratio=0.319305 router_tokens_per_expert=[515, 538, 517, 498, 511, 525, 523, 495, 519, 482, 516, 515, 523, 517, 517, 514, 508, 473, 535, 520, 481, 513, 494, 461, 519, 481, 482, 515, 496, 533, 504, 509, 510, 496, 562, 540, 555, 527, 503, 511, 522, 515, 509, 508, 548, 486, 503, 485, 519, 544, 513, 513, 536, 492, 492, 487, 483, 511, 570, 533, 521, 527, 483, 515] payload_rows_per_dst=[5478, 5716, 5474, 5548, 5660, 5524, 5562, 5648] dup_rows_per_dst=[2514, 2742, 2496, 2530, 2748, 2628, 2592, 2676] combine_rows_per_src=[5478, 5716, 5474, 5548, 5660, 5524, 5562, 5648] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44488 payload_rows_remote=38994 dup_rows=21048 dup_rows_remote=18450 dedup_ratio=0.321167 router_tokens_per_expert=[508, 542, 531, 526, 493, 496, 530, 491, 516, 468, 493, 512, 530, 526, 506, 495, 506, 506, 524, 474, 524, 547, 559, 485, 525, 536, 513, 522, 495, 543, 514, 521, 530, 489, 495, 545, 544, 493, 521, 509, 538, 487, 522, 521, 501, 529, 547, 514, 498, 500, 523, 476, 502, 524, 522, 509, 504, 501, 492, 464, 504, 488, 493, 526] payload_rows_per_dst=[5584, 5494, 5554, 5588, 5562, 5682, 5558, 5466] dup_rows_per_dst=[2650, 2598, 2696, 2750, 2690, 2636, 2550, 2478] combine_rows_per_src=[5584, 5494, 5554, 5588, 5562, 5682, 5558, 5466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38858 dup_rows=20926 dup_rows_remote=18282 dedup_ratio=0.319305 router_tokens_per_expert=[516, 496, 494, 515, 550, 476, 512, 571, 512, 508, 504, 479, 479, 529, 526, 490, 498, 486, 509, 526, 537, 504, 499, 522, 499, 516, 515, 507, 504, 497, 525, 525, 516, 549, 496, 494, 519, 532, 492, 481, 525, 506, 522, 512, 508, 520, 536, 491, 513, 509, 538, 559, 458, 551, 560, 518, 516, 492, 541, 486, 463, 525, 481, 533] payload_rows_per_dst=[5752, 5528, 5794, 5366, 5696, 5468, 5590, 5416] dup_rows_per_dst=[2644, 2526, 2912, 2388, 2758, 2372, 2790, 2536] combine_rows_per_src=[5752, 5528, 5794, 5366, 5696, 5468, 5590, 5416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38848 dup_rows=21084 dup_rows_remote=18490 dedup_ratio=0.321716 router_tokens_per_expert=[523, 536, 562, 509, 499, 474, 478, 479, 563, 507, 510, 508, 534, 519, 530, 516, 521, 552, 517, 505, 518, 501, 499, 469, 526, 479, 497, 518, 508, 543, 539, 530, 525, 502, 442, 545, 514, 535, 499, 502, 505, 512, 555, 531, 497, 531, 469, 499, 484, 510, 523, 500, 554, 531, 475, 495, 504, 552, 476, 517, 515, 502, 491, 507] payload_rows_per_dst=[5626, 5624, 5510, 5550, 5592, 5604, 5476, 5470] dup_rows_per_dst=[2494, 2750, 2654, 2730, 2536, 2594, 2668, 2658] combine_rows_per_src=[5626, 5624, 5510, 5550, 5592, 5604, 5476, 5470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=39142 dup_rows=21008 dup_rows_remote=18506 dedup_ratio=0.320557 router_tokens_per_expert=[527, 529, 527, 564, 515, 484, 480, 511, 552, 506, 496, 484, 537, 491, 496, 497, 538, 477, 491, 507, 502, 472, 519, 492, 480, 532, 515, 539, 547, 519, 539, 515, 488, 440, 497, 474, 509, 519, 494, 523, 511, 515, 548, 543, 539, 493, 509, 500, 514, 526, 485, 482, 530, 531, 534, 522, 553, 507, 512, 491, 486, 523, 527, 563] payload_rows_per_dst=[5598, 5502, 5516, 5636, 5386, 5632, 5622, 5636] dup_rows_per_dst=[2676, 2616, 2480, 2736, 2502, 2684, 2626, 2688] combine_rows_per_src=[5598, 5502, 5516, 5636, 5386, 5632, 5622, 5636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=228848 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279508464 ingress_bytes=499480576 link_bytes=499480576 token_bytes=558788080 weight_movement_bytes=220200960 total_nvlink_bytes=778989040 weight_to_token_ratio=0.394069 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686788721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=229288 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278648744 ingress_bytes=608720896 link_bytes=608720896 token_bytes=557068200 weight_movement_bytes=330301440 total_nvlink_bytes=887369640 weight_to_token_ratio=0.592928 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058665756e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38988 dup_rows=20976 dup_rows_remote=18408 dedup_ratio=0.320068 router_tokens_per_expert=[460, 523, 495, 549, 527, 473, 502, 488, 493, 518, 510, 491, 551, 499, 489, 519, 510, 501, 526, 505, 539, 470, 479, 521, 509, 514, 484, 535, 511, 507, 522, 591, 496, 502, 493, 487, 550, 510, 531, 501, 540, 514, 472, 562, 520, 543, 487, 517, 461, 494, 475, 526, 519, 519, 558, 528, 572, 496, 481, 500, 523, 519, 530, 531] payload_rows_per_dst=[5480, 5572, 5498, 5620, 5546, 5652, 5574, 5618] dup_rows_per_dst=[2554, 2568, 2604, 2726, 2594, 2658, 2586, 2686] combine_rows_per_src=[5480, 5572, 5498, 5620, 5546, 5652, 5574, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279939072 dispatch_meta_bytes=229320 combine_payload_bytes=279939072 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=280168392 ingress_bytes=720340992 link_bytes=720340992 token_bytes=560107464 weight_movement_bytes=440401920 total_nvlink_bytes=1000509384 weight_to_token_ratio=0.786281 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252791461e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=38964 dup_rows=20948 dup_rows_remote=18254 dedup_ratio=0.319641 router_tokens_per_expert=[498, 478, 496, 493, 542, 512, 510, 464, 509, 513, 517, 542, 454, 553, 527, 512, 522, 507, 508, 549, 493, 522, 545, 513, 498, 544, 473, 517, 527, 539, 509, 528, 517, 519, 520, 477, 488, 523, 522, 496, 536, 494, 501, 523, 535, 501, 517, 553, 506, 507, 544, 492, 504, 490, 514, 478, 491, 513, 513, 479, 549, 528, 517, 507] payload_rows_per_dst=[5448, 5540, 5624, 5598, 5656, 5662, 5480, 5580] dup_rows_per_dst=[2538, 2714, 2694, 2672, 2468, 2658, 2590, 2614] combine_rows_per_src=[5448, 5540, 5624, 5598, 5656, 5662, 5480, 5580] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44432 payload_rows_remote=38872 dup_rows=21104 dup_rows_remote=18386 dedup_ratio=0.322021 router_tokens_per_expert=[469, 495, 548, 503, 503, 510, 528, 535, 500, 508, 526, 488, 524, 523, 544, 484, 515, 514, 524, 518, 491, 528, 492, 508, 561, 524, 521, 520, 516, 503, 510, 526, 504, 482, 544, 511, 520, 516, 451, 519, 503, 500, 498, 483, 515, 480, 514, 528, 539, 518, 558, 527, 527, 500, 473, 497, 496, 543, 487, 544, 509, 471, 511, 541] payload_rows_per_dst=[5532, 5586, 5496, 5654, 5540, 5548, 5560, 5516] dup_rows_per_dst=[2650, 2608, 2684, 2708, 2554, 2494, 2718, 2688] combine_rows_per_src=[5532, 5586, 5496, 5654, 5540, 5548, 5560, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=39142 dup_rows=21008 dup_rows_remote=18506 dedup_ratio=0.320557 router_tokens_per_expert=[527, 529, 527, 564, 515, 484, 480, 511, 552, 506, 496, 484, 537, 491, 496, 497, 538, 477, 491, 507, 502, 472, 519, 492, 480, 532, 515, 539, 547, 519, 539, 515, 488, 440, 497, 474, 509, 519, 494, 523, 511, 515, 548, 543, 539, 493, 509, 500, 514, 526, 485, 482, 530, 531, 534, 522, 553, 507, 512, 491, 486, 523, 527, 563] payload_rows_per_dst=[5598, 5502, 5516, 5636, 5386, 5632, 5622, 5636] dup_rows_per_dst=[2676, 2616, 2480, 2736, 2502, 2684, 2626, 2688] combine_rows_per_src=[5598, 5502, 5516, 5636, 5386, 5632, 5622, 5636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38858 dup_rows=20926 dup_rows_remote=18282 dedup_ratio=0.319305 router_tokens_per_expert=[516, 496, 494, 515, 550, 476, 512, 571, 512, 508, 504, 479, 479, 529, 526, 490, 498, 486, 509, 526, 537, 504, 499, 522, 499, 516, 515, 507, 504, 497, 525, 525, 516, 549, 496, 494, 519, 532, 492, 481, 525, 506, 522, 512, 508, 520, 536, 491, 513, 509, 538, 559, 458, 551, 560, 518, 516, 492, 541, 486, 463, 525, 481, 533] payload_rows_per_dst=[5752, 5528, 5794, 5366, 5696, 5468, 5590, 5416] dup_rows_per_dst=[2644, 2526, 2912, 2388, 2758, 2372, 2790, 2536] combine_rows_per_src=[5752, 5528, 5794, 5366, 5696, 5468, 5590, 5416] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44272 payload_rows_remote=38724 dup_rows=21264 dup_rows_remote=18594 dedup_ratio=0.324463 router_tokens_per_expert=[482, 499, 512, 552, 532, 519, 501, 483, 524, 515, 506, 514, 481, 485, 499, 509, 540, 514, 504, 541, 510, 495, 509, 452, 501, 545, 531, 503, 496, 500, 520, 513, 515, 504, 526, 495, 490, 522, 514, 493, 523, 550, 503, 512, 542, 500, 517, 502, 503, 503, 518, 478, 550, 502, 534, 494, 516, 514, 537, 510, 537, 524, 534, 519] payload_rows_per_dst=[5534, 5456, 5550, 5548, 5444, 5592, 5522, 5626] dup_rows_per_dst=[2626, 2610, 2580, 2670, 2674, 2706, 2642, 2756] combine_rows_per_src=[5534, 5456, 5550, 5548, 5444, 5592, 5522, 5626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=229776 combine_payload_bytes=279508992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279738768 ingress_bytes=389609472 link_bytes=389609472 token_bytes=559247760 weight_movement_bytes=110100480 total_nvlink_bytes=669348240 weight_to_token_ratio=0.196872 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775949516e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=228560 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278762704 ingress_bytes=498735104 link_bytes=498735104 token_bytes=557296848 weight_movement_bytes=220200960 total_nvlink_bytes=777497808 weight_to_token_ratio=0.395123 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673823737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280569856 dispatch_meta_bytes=230592 combine_payload_bytes=280569856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280800448 ingress_bytes=390670336 link_bytes=390670336 token_bytes=561370304 weight_movement_bytes=110100480 total_nvlink_bytes=671470784 weight_to_token_ratio=0.196128 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794399685e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44584 payload_rows_remote=38988 dup_rows=20952 dup_rows_remote=18408 dedup_ratio=0.319702 router_tokens_per_expert=[517, 516, 486, 481, 519, 541, 539, 490, 515, 521, 501, 511, 542, 510, 491, 499, 509, 514, 512, 507, 523, 501, 516, 488, 506, 525, 514, 511, 500, 490, 527, 494, 505, 495, 521, 494, 518, 527, 518, 519, 523, 499, 554, 535, 498, 495, 529, 553, 541, 513, 519, 493, 488, 489, 483, 504, 511, 518, 539, 529, 503, 510, 528, 501] payload_rows_per_dst=[5586, 5506, 5596, 5550, 5534, 5642, 5510, 5660] dup_rows_per_dst=[2592, 2674, 2544, 2584, 2660, 2730, 2550, 2618] combine_rows_per_src=[5586, 5506, 5596, 5550, 5534, 5642, 5510, 5660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=229352 combine_payload_bytes=278462464 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278691816 ingress_bytes=608763904 link_bytes=608763904 token_bytes=557154280 weight_movement_bytes=330301440 total_nvlink_bytes=887455720 weight_to_token_ratio=0.592837 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058740554e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=229584 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279695568 ingress_bytes=389566464 link_bytes=389566464 token_bytes=559161552 weight_movement_bytes=110100480 total_nvlink_bytes=669262032 weight_to_token_ratio=0.196903 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775201536e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38848 dup_rows=21084 dup_rows_remote=18490 dedup_ratio=0.321716 router_tokens_per_expert=[523, 536, 562, 509, 499, 474, 478, 479, 563, 507, 510, 508, 534, 519, 530, 516, 521, 552, 517, 505, 518, 501, 499, 469, 526, 479, 497, 518, 508, 543, 539, 530, 525, 502, 442, 545, 514, 535, 499, 502, 505, 512, 555, 531, 497, 531, 469, 499, 484, 510, 523, 500, 554, 531, 475, 495, 504, 552, 476, 517, 515, 502, 491, 507] payload_rows_per_dst=[5626, 5624, 5510, 5550, 5592, 5604, 5476, 5470] dup_rows_per_dst=[2494, 2750, 2654, 2730, 2536, 2594, 2668, 2658] combine_rows_per_src=[5626, 5624, 5510, 5550, 5592, 5604, 5476, 5470] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=229032 combine_payload_bytes=278634496 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278863528 ingress_bytes=498835456 link_bytes=498835456 token_bytes=557498024 weight_movement_bytes=220200960 total_nvlink_bytes=777698984 weight_to_token_ratio=0.394981 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.675569023e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=228560 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278762704 ingress_bytes=498735104 link_bytes=498735104 token_bytes=557296848 weight_movement_bytes=220200960 total_nvlink_bytes=777497808 weight_to_token_ratio=0.395123 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673823737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280569856 dispatch_meta_bytes=230592 combine_payload_bytes=280569856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280800448 ingress_bytes=390670336 link_bytes=390670336 token_bytes=561370304 weight_movement_bytes=110100480 total_nvlink_bytes=671470784 weight_to_token_ratio=0.196128 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794399685e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=229272 combine_payload_bytes=277573632 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277802904 ingress_bytes=497774592 link_bytes=497774592 token_bytes=555376536 weight_movement_bytes=220200960 total_nvlink_bytes=775577496 weight_to_token_ratio=0.396489 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.657118853e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=229584 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279695568 ingress_bytes=389566464 link_bytes=389566464 token_bytes=559161552 weight_movement_bytes=110100480 total_nvlink_bytes=669262032 weight_to_token_ratio=0.196903 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775201536e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38724 dup_rows=21150 dup_rows_remote=18342 dedup_ratio=0.322723 router_tokens_per_expert=[478, 515, 466, 536, 493, 523, 510, 479, 497, 564, 541, 527, 455, 521, 559, 493, 551, 519, 525, 535, 519, 492, 542, 508, 486, 554, 529, 494, 482, 559, 497, 493, 523, 494, 491, 512, 485, 492, 512, 525, 531, 464, 504, 504, 490, 479, 562, 500, 511, 505, 519, 532, 489, 519, 519, 535, 505, 538, 504, 543, 494, 511, 531, 503] payload_rows_per_dst=[5470, 5590, 5432, 5466, 5554, 5570, 5642, 5662] dup_rows_per_dst=[2530, 2720, 2586, 2532, 2658, 2576, 2740, 2808] combine_rows_per_src=[5470, 5590, 5432, 5466, 5554, 5570, 5642, 5662] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44514 payload_rows_remote=38970 dup_rows=21022 dup_rows_remote=18452 dedup_ratio=0.320770 router_tokens_per_expert=[513, 466, 566, 529, 519, 502, 516, 513, 522, 512, 510, 497, 510, 510, 484, 512, 484, 570, 499, 511, 500, 512, 529, 507, 513, 548, 512, 509, 520, 535, 490, 495, 493, 527, 527, 511, 532, 536, 506, 524, 510, 509, 550, 492, 464, 515, 504, 495, 489, 485, 548, 513, 521, 493, 567, 537, 457, 503, 494, 530, 496, 515, 508, 502] payload_rows_per_dst=[5630, 5544, 5558, 5510, 5578, 5562, 5598, 5534] dup_rows_per_dst=[2618, 2570, 2666, 2734, 2734, 2516, 2708, 2476] combine_rows_per_src=[5630, 5544, 5558, 5510, 5578, 5562, 5598, 5534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38958 dup_rows=20992 dup_rows_remote=18418 dedup_ratio=0.320312 router_tokens_per_expert=[532, 527, 524, 503, 533, 489, 508, 518, 548, 510, 474, 511, 487, 509, 495, 546, 497, 499, 533, 514, 525, 493, 503, 505, 511, 502, 503, 528, 524, 482, 499, 499, 491, 536, 515, 471, 512, 544, 515, 496, 516, 518, 550, 515, 519, 501, 509, 525, 508, 520, 485, 517, 518, 520, 512, 505, 532, 491, 521, 541, 487, 530, 504, 513] payload_rows_per_dst=[5612, 5616, 5522, 5508, 5586, 5540, 5544, 5616] dup_rows_per_dst=[2656, 2544, 2616, 2588, 2574, 2766, 2626, 2622] combine_rows_per_src=[5612, 5616, 5522, 5508, 5586, 5540, 5544, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=228872 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279522824 ingress_bytes=499494912 link_bytes=499494912 token_bytes=558816776 weight_movement_bytes=220200960 total_nvlink_bytes=779017736 weight_to_token_ratio=0.394049 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687038048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=229352 combine_payload_bytes=278462464 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278691816 ingress_bytes=608763904 link_bytes=608763904 token_bytes=557154280 weight_movement_bytes=330301440 total_nvlink_bytes=887455720 weight_to_token_ratio=0.592837 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058740554e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38962 dup_rows=20926 dup_rows_remote=18250 dedup_ratio=0.319305 router_tokens_per_expert=[515, 538, 517, 498, 511, 525, 523, 495, 519, 482, 516, 515, 523, 517, 517, 514, 508, 473, 535, 520, 481, 513, 494, 461, 519, 481, 482, 515, 496, 533, 504, 509, 510, 496, 562, 540, 555, 527, 503, 511, 522, 515, 509, 508, 548, 486, 503, 485, 519, 544, 513, 513, 536, 492, 492, 487, 483, 511, 570, 533, 521, 527, 483, 515] payload_rows_per_dst=[5478, 5716, 5474, 5548, 5660, 5524, 5562, 5648] dup_rows_per_dst=[2514, 2742, 2496, 2530, 2748, 2628, 2592, 2676] combine_rows_per_src=[5478, 5716, 5474, 5548, 5660, 5524, 5562, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44540 payload_rows_remote=38884 dup_rows=20996 dup_rows_remote=18314 dedup_ratio=0.320374 router_tokens_per_expert=[529, 510, 521, 502, 508, 522, 504, 492, 506, 507, 496, 476, 535, 534, 515, 515, 560, 548, 483, 532, 513, 547, 512, 517, 510, 525, 512, 475, 504, 472, 527, 532, 539, 478, 526, 483, 495, 523, 519, 514, 514, 517, 518, 527, 545, 484, 501, 518, 497, 502, 538, 509, 492, 500, 515, 517, 505, 560, 526, 484, 504, 508, 498, 471] payload_rows_per_dst=[5656, 5624, 5556, 5514, 5528, 5620, 5516, 5526] dup_rows_per_dst=[2682, 2544, 2706, 2600, 2626, 2628, 2624, 2586] combine_rows_per_src=[5656, 5624, 5556, 5514, 5528, 5620, 5516, 5526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38964 dup_rows=21042 dup_rows_remote=18406 dedup_ratio=0.321075 router_tokens_per_expert=[463, 493, 473, 500, 489, 524, 537, 530, 537, 485, 486, 470, 527, 550, 519, 516, 499, 467, 479, 529, 540, 492, 512, 526, 500, 525, 482, 542, 548, 533, 539, 518, 525, 470, 517, 476, 524, 531, 502, 521, 535, 515, 518, 482, 546, 502, 500, 485, 532, 498, 533, 505, 500, 499, 542, 544, 526, 513, 497, 511, 520, 537, 521, 511] payload_rows_per_dst=[5466, 5548, 5574, 5664, 5530, 5530, 5578, 5604] dup_rows_per_dst=[2552, 2632, 2514, 2710, 2602, 2636, 2728, 2668] combine_rows_per_src=[5466, 5548, 5574, 5664, 5530, 5530, 5578, 5604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=228264 combine_payload_bytes=277573632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277801896 ingress_bytes=387674112 link_bytes=387674112 token_bytes=555375528 weight_movement_bytes=110100480 total_nvlink_bytes=665476008 weight_to_token_ratio=0.198245 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.742290422e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44616 payload_rows_remote=39044 dup_rows=20920 dup_rows_remote=18406 dedup_ratio=0.319214 router_tokens_per_expert=[494, 531, 544, 524, 527, 473, 540, 568, 484, 497, 499, 545, 511, 545, 490, 527, 511, 519, 489, 534, 518, 505, 521, 516, 494, 496, 516, 535, 496, 506, 536, 544, 476, 497, 527, 501, 516, 514, 491, 538, 490, 519, 518, 498, 512, 534, 533, 517, 502, 534, 482, 528, 453, 514, 517, 513, 481, 485, 493, 534, 505, 510, 498, 503] payload_rows_per_dst=[5610, 5596, 5598, 5604, 5486, 5636, 5572, 5514] dup_rows_per_dst=[2792, 2600, 2628, 2642, 2634, 2606, 2514, 2504] combine_rows_per_src=[5610, 5596, 5598, 5604, 5486, 5636, 5572, 5514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=229688 combine_payload_bytes=279336960 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279566648 ingress_bytes=389437440 link_bytes=389437440 token_bytes=558903608 weight_movement_bytes=110100480 total_nvlink_bytes=669004088 weight_to_token_ratio=0.196994 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772957596e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44488 payload_rows_remote=38994 dup_rows=21048 dup_rows_remote=18450 dedup_ratio=0.321167 router_tokens_per_expert=[508, 542, 531, 526, 493, 496, 530, 491, 516, 468, 493, 512, 530, 526, 506, 495, 506, 506, 524, 474, 524, 547, 559, 485, 525, 536, 513, 522, 495, 543, 514, 521, 530, 489, 495, 545, 544, 493, 521, 509, 538, 487, 522, 521, 501, 529, 547, 514, 498, 500, 523, 476, 502, 524, 522, 509, 504, 501, 492, 464, 504, 488, 493, 526] payload_rows_per_dst=[5584, 5494, 5554, 5588, 5562, 5682, 5558, 5466] dup_rows_per_dst=[2650, 2598, 2696, 2750, 2690, 2636, 2550, 2478] combine_rows_per_src=[5584, 5494, 5554, 5588, 5562, 5682, 5558, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=228848 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279508464 ingress_bytes=499480576 link_bytes=499480576 token_bytes=558788080 weight_movement_bytes=220200960 total_nvlink_bytes=778989040 weight_to_token_ratio=0.394069 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686788721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38958 dup_rows=20992 dup_rows_remote=18418 dedup_ratio=0.320312 router_tokens_per_expert=[532, 527, 524, 503, 533, 489, 508, 518, 548, 510, 474, 511, 487, 509, 495, 546, 497, 499, 533, 514, 525, 493, 503, 505, 511, 502, 503, 528, 524, 482, 499, 499, 491, 536, 515, 471, 512, 544, 515, 496, 516, 518, 550, 515, 519, 501, 509, 525, 508, 520, 485, 517, 518, 520, 512, 505, 532, 491, 521, 541, 487, 530, 504, 513] payload_rows_per_dst=[5612, 5616, 5522, 5508, 5586, 5540, 5544, 5616] dup_rows_per_dst=[2656, 2544, 2616, 2588, 2574, 2766, 2626, 2622] combine_rows_per_src=[5612, 5616, 5522, 5508, 5586, 5540, 5544, 5616] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38964 dup_rows=20978 dup_rows_remote=18318 dedup_ratio=0.320099 router_tokens_per_expert=[494, 514, 502, 526, 500, 476, 475, 534, 504, 503, 523, 502, 480, 499, 544, 518, 511, 506, 534, 496, 503, 540, 505, 520, 496, 581, 517, 521, 510, 494, 496, 512, 520, 518, 496, 512, 549, 511, 525, 522, 502, 496, 512, 514, 502, 545, 505, 511, 513, 507, 521, 511, 493, 493, 515, 493, 500, 542, 519, 541, 519, 538, 508, 479] payload_rows_per_dst=[5518, 5584, 5570, 5594, 5618, 5568, 5478, 5628] dup_rows_per_dst=[2524, 2562, 2660, 2660, 2688, 2606, 2614, 2664] combine_rows_per_src=[5518, 5584, 5570, 5594, 5618, 5568, 5478, 5628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=229504 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279480448 ingress_bytes=389351424 link_bytes=389351424 token_bytes=558731392 weight_movement_bytes=110100480 total_nvlink_bytes=668831872 weight_to_token_ratio=0.197054 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771461636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=228792 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278949304 ingress_bytes=388820992 link_bytes=388820992 token_bytes=557669816 weight_movement_bytes=110100480 total_nvlink_bytes=667770296 weight_to_token_ratio=0.197430 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762236552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229480 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279523432 ingress_bytes=389394432 link_bytes=389394432 token_bytes=558817384 weight_movement_bytes=110100480 total_nvlink_bytes=668917864 weight_to_token_ratio=0.197024 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772209616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=38932 dup_rows=20990 dup_rows_remote=18254 dedup_ratio=0.320282 router_tokens_per_expert=[509, 515, 524, 523, 508, 498, 506, 527, 540, 518, 542, 505, 518, 499, 536, 473, 490, 512, 539, 516, 507, 521, 550, 540, 512, 503, 472, 507, 533, 552, 527, 499, 528, 482, 496, 535, 529, 504, 523, 473, 479, 508, 516, 490, 536, 505, 514, 540, 517, 508, 491, 483, 485, 514, 522, 507, 511, 538, 524, 527, 511, 495, 481, 475] payload_rows_per_dst=[5618, 5534, 5614, 5606, 5546, 5654, 5486, 5488] dup_rows_per_dst=[2602, 2728, 2736, 2604, 2594, 2522, 2568, 2636] combine_rows_per_src=[5618, 5534, 5614, 5606, 5546, 5654, 5486, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38964 dup_rows=21042 dup_rows_remote=18406 dedup_ratio=0.321075 router_tokens_per_expert=[463, 493, 473, 500, 489, 524, 537, 530, 537, 485, 486, 470, 527, 550, 519, 516, 499, 467, 479, 529, 540, 492, 512, 526, 500, 525, 482, 542, 548, 533, 539, 518, 525, 470, 517, 476, 524, 531, 502, 521, 535, 515, 518, 482, 546, 502, 500, 485, 532, 498, 533, 505, 500, 499, 542, 544, 526, 513, 497, 511, 520, 537, 521, 511] payload_rows_per_dst=[5466, 5548, 5574, 5664, 5530, 5530, 5578, 5604] dup_rows_per_dst=[2552, 2632, 2514, 2710, 2602, 2636, 2728, 2668] combine_rows_per_src=[5466, 5548, 5574, 5664, 5530, 5530, 5578, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=228792 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278949304 ingress_bytes=388820992 link_bytes=388820992 token_bytes=557669816 weight_movement_bytes=110100480 total_nvlink_bytes=667770296 weight_to_token_ratio=0.197430 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762236552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279867392 dispatch_meta_bytes=229800 combine_payload_bytes=279867392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280097192 ingress_bytes=389967872 link_bytes=389967872 token_bytes=559964584 weight_movement_bytes=110100480 total_nvlink_bytes=670065064 weight_to_token_ratio=0.196620 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782182681e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=229776 combine_payload_bytes=279508992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279738768 ingress_bytes=389609472 link_bytes=389609472 token_bytes=559247760 weight_movement_bytes=110100480 total_nvlink_bytes=669348240 weight_to_token_ratio=0.196872 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775949516e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=229504 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279480448 ingress_bytes=389351424 link_bytes=389351424 token_bytes=558731392 weight_movement_bytes=110100480 total_nvlink_bytes=668831872 weight_to_token_ratio=0.197054 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771461636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229128 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279523080 ingress_bytes=609595392 link_bytes=609595392 token_bytes=558817032 weight_movement_bytes=330301440 total_nvlink_bytes=889118472 weight_to_token_ratio=0.591073 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060186648e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=228744 combine_payload_bytes=279064576 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=279293320 ingress_bytes=829566976 link_bytes=829566976 token_bytes=558357896 weight_movement_bytes=550502400 total_nvlink_bytes=1108860296 weight_to_token_ratio=0.985931 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.442753412e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38710 dup_rows=21142 dup_rows_remote=18214 dedup_ratio=0.322601 router_tokens_per_expert=[511, 523, 539, 520, 468, 529, 557, 539, 525, 478, 539, 500, 511, 487, 550, 511, 512, 508, 508, 519, 506, 469, 517, 528, 542, 486, 487, 505, 509, 522, 490, 506, 499, 542, 518, 505, 527, 529, 482, 533, 504, 480, 499, 517, 526, 483, 505, 500, 496, 536, 507, 504, 498, 531, 525, 496, 516, 478, 519, 519, 518, 525, 531, 519] payload_rows_per_dst=[5516, 5592, 5580, 5594, 5538, 5330, 5560, 5684] dup_rows_per_dst=[2552, 2610, 2554, 2748, 2732, 2392, 2626, 2928] combine_rows_per_src=[5516, 5592, 5580, 5594, 5538, 5330, 5560, 5684] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44356 payload_rows_remote=38748 dup_rows=21180 dup_rows_remote=18592 dedup_ratio=0.323181 router_tokens_per_expert=[496, 497, 532, 508, 501, 503, 502, 527, 555, 523, 485, 475, 518, 487, 556, 499, 503, 527, 519, 522, 512, 499, 504, 496, 506, 527, 512, 515, 547, 511, 495, 537, 491, 529, 502, 505, 546, 523, 451, 554, 547, 504, 529, 534, 524, 480, 504, 540, 501, 537, 498, 524, 494, 497, 554, 500, 489, 517, 517, 485, 529, 494, 487, 486] payload_rows_per_dst=[5538, 5608, 5542, 5598, 5494, 5574, 5576, 5426] dup_rows_per_dst=[2594, 2588, 2622, 2702, 2708, 2750, 2634, 2582] combine_rows_per_src=[5538, 5608, 5542, 5598, 5494, 5574, 5576, 5426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39014 dup_rows=20988 dup_rows_remote=18360 dedup_ratio=0.320251 router_tokens_per_expert=[524, 468, 540, 543, 526, 525, 519, 499, 483, 518, 509, 505, 523, 544, 530, 526, 514, 547, 511, 517, 489, 494, 533, 514, 521, 536, 527, 534, 526, 470, 532, 481, 519, 505, 496, 538, 535, 478, 522, 488, 523, 522, 530, 540, 478, 495, 507, 502, 486, 530, 512, 493, 492, 525, 488, 487, 513, 492, 483, 499, 484, 563, 499, 516] payload_rows_per_dst=[5552, 5626, 5636, 5636, 5534, 5562, 5466, 5536] dup_rows_per_dst=[2736, 2650, 2602, 2618, 2628, 2632, 2560, 2562] combine_rows_per_src=[5552, 5626, 5636, 5636, 5534, 5562, 5466, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44584 payload_rows_remote=38988 dup_rows=20952 dup_rows_remote=18408 dedup_ratio=0.319702 router_tokens_per_expert=[517, 516, 486, 481, 519, 541, 539, 490, 515, 521, 501, 511, 542, 510, 491, 499, 509, 514, 512, 507, 523, 501, 516, 488, 506, 525, 514, 511, 500, 490, 527, 494, 505, 495, 521, 494, 518, 527, 518, 519, 523, 499, 554, 535, 498, 495, 529, 553, 541, 513, 519, 493, 488, 489, 483, 504, 511, 518, 539, 529, 503, 510, 528, 501] payload_rows_per_dst=[5586, 5506, 5596, 5550, 5534, 5642, 5510, 5660] dup_rows_per_dst=[2592, 2674, 2544, 2584, 2660, 2730, 2550, 2618] combine_rows_per_src=[5586, 5506, 5596, 5550, 5534, 5642, 5510, 5660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229480 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279523432 ingress_bytes=389394432 link_bytes=389394432 token_bytes=558817384 weight_movement_bytes=110100480 total_nvlink_bytes=668917864 weight_to_token_ratio=0.197024 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772209616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38724 dup_rows=21150 dup_rows_remote=18342 dedup_ratio=0.322723 router_tokens_per_expert=[478, 515, 466, 536, 493, 523, 510, 479, 497, 564, 541, 527, 455, 521, 559, 493, 551, 519, 525, 535, 519, 492, 542, 508, 486, 554, 529, 494, 482, 559, 497, 493, 523, 494, 491, 512, 485, 492, 512, 525, 531, 464, 504, 504, 490, 479, 562, 500, 511, 505, 519, 532, 489, 519, 519, 535, 505, 538, 504, 543, 494, 511, 531, 503] payload_rows_per_dst=[5470, 5590, 5432, 5466, 5554, 5570, 5642, 5662] dup_rows_per_dst=[2530, 2720, 2586, 2532, 2658, 2576, 2740, 2808] combine_rows_per_src=[5470, 5590, 5432, 5466, 5554, 5570, 5642, 5662] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=229584 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279695568 ingress_bytes=389566464 link_bytes=389566464 token_bytes=559161552 weight_movement_bytes=110100480 total_nvlink_bytes=669262032 weight_to_token_ratio=0.196903 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775201536e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=39080 dup_rows=20940 dup_rows_remote=18304 dedup_ratio=0.319519 router_tokens_per_expert=[527, 520, 533, 524, 518, 494, 534, 514, 486, 538, 551, 473, 496, 508, 533, 498, 485, 501, 524, 508, 485, 489, 519, 497, 518, 496, 521, 474, 501, 509, 529, 538, 545, 506, 511, 494, 536, 510, 499, 555, 524, 518, 520, 523, 498, 516, 485, 512, 509, 502, 534, 493, 455, 515, 553, 515, 499, 521, 527, 542, 522, 490, 523, 475] payload_rows_per_dst=[5604, 5612, 5472, 5594, 5614, 5612, 5516, 5572] dup_rows_per_dst=[2724, 2554, 2544, 2578, 2698, 2580, 2636, 2626] combine_rows_per_src=[5604, 5612, 5472, 5594, 5614, 5612, 5516, 5572] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38758 dup_rows=21150 dup_rows_remote=18390 dedup_ratio=0.322723 router_tokens_per_expert=[513, 525, 514, 509, 528, 542, 526, 526, 517, 488, 540, 515, 503, 537, 505, 526, 472, 507, 487, 543, 524, 505, 533, 495, 478, 477, 464, 515, 504, 535, 519, 500, 536, 520, 497, 505, 486, 517, 497, 523, 492, 505, 521, 514, 551, 532, 539, 523, 542, 491, 485, 523, 507, 468, 542, 496, 478, 548, 484, 533, 483, 483, 562, 513] payload_rows_per_dst=[5628, 5570, 5578, 5460, 5524, 5492, 5518, 5616] dup_rows_per_dst=[2760, 2676, 2676, 2524, 2638, 2740, 2584, 2552] combine_rows_per_src=[5628, 5570, 5578, 5460, 5524, 5492, 5518, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=228264 combine_payload_bytes=277573632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277801896 ingress_bytes=387674112 link_bytes=387674112 token_bytes=555375528 weight_movement_bytes=110100480 total_nvlink_bytes=665476008 weight_to_token_ratio=0.198245 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.742290422e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44706 payload_rows_remote=39156 dup_rows=20830 dup_rows_remote=18262 dedup_ratio=0.317841 router_tokens_per_expert=[522, 493, 494, 519, 548, 493, 511, 471, 513, 517, 520, 519, 483, 523, 500, 517, 523, 510, 515, 500, 565, 513, 504, 504, 490, 524, 513, 519, 486, 523, 498, 506, 520, 502, 529, 524, 510, 511, 478, 500, 481, 537, 509, 504, 514, 509, 516, 517, 522, 481, 543, 525, 503, 504, 542, 497, 510, 533, 527, 530, 552, 537, 482, 483] payload_rows_per_dst=[5576, 5576, 5616, 5550, 5558, 5630, 5588, 5612] dup_rows_per_dst=[2526, 2608, 2652, 2568, 2590, 2544, 2646, 2696] combine_rows_per_src=[5576, 5576, 5616, 5550, 5558, 5630, 5588, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44514 payload_rows_remote=38970 dup_rows=21022 dup_rows_remote=18452 dedup_ratio=0.320770 router_tokens_per_expert=[513, 466, 566, 529, 519, 502, 516, 513, 522, 512, 510, 497, 510, 510, 484, 512, 484, 570, 499, 511, 500, 512, 529, 507, 513, 548, 512, 509, 520, 535, 490, 495, 493, 527, 527, 511, 532, 536, 506, 524, 510, 509, 550, 492, 464, 515, 504, 495, 489, 485, 548, 513, 521, 493, 567, 537, 457, 503, 494, 530, 496, 515, 508, 502] payload_rows_per_dst=[5630, 5544, 5558, 5510, 5578, 5562, 5598, 5534] dup_rows_per_dst=[2618, 2570, 2666, 2734, 2734, 2516, 2708, 2476] combine_rows_per_src=[5630, 5544, 5558, 5510, 5578, 5562, 5598, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39014 dup_rows=20988 dup_rows_remote=18360 dedup_ratio=0.320251 router_tokens_per_expert=[524, 468, 540, 543, 526, 525, 519, 499, 483, 518, 509, 505, 523, 544, 530, 526, 514, 547, 511, 517, 489, 494, 533, 514, 521, 536, 527, 534, 526, 470, 532, 481, 519, 505, 496, 538, 535, 478, 522, 488, 523, 522, 530, 540, 478, 495, 507, 502, 486, 530, 512, 493, 492, 525, 488, 487, 513, 492, 483, 499, 484, 563, 499, 516] payload_rows_per_dst=[5552, 5626, 5636, 5636, 5534, 5562, 5466, 5536] dup_rows_per_dst=[2736, 2650, 2602, 2618, 2628, 2632, 2560, 2562] combine_rows_per_src=[5552, 5626, 5636, 5636, 5534, 5562, 5466, 5536] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44508 payload_rows_remote=38952 dup_rows=21028 dup_rows_remote=18278 dedup_ratio=0.320862 router_tokens_per_expert=[501, 530, 517, 500, 521, 474, 495, 497, 462, 554, 495, 520, 500, 517, 516, 502, 547, 513, 513, 492, 487, 502, 500, 539, 541, 523, 524, 504, 491, 536, 521, 502, 533, 506, 501, 499, 534, 489, 506, 474, 527, 517, 548, 528, 515, 480, 490, 548, 523, 508, 517, 559, 531, 531, 509, 540, 482, 496, 506, 527, 508, 501, 511, 488] payload_rows_per_dst=[5514, 5582, 5578, 5582, 5520, 5556, 5728, 5448] dup_rows_per_dst=[2556, 2550, 2608, 2702, 2564, 2750, 2708, 2590] combine_rows_per_src=[5514, 5582, 5578, 5582, 5520, 5556, 5728, 5448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277817344 dispatch_meta_bytes=228592 combine_payload_bytes=277817344 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278045936 ingress_bytes=608118784 link_bytes=608118784 token_bytes=555863280 weight_movement_bytes=330301440 total_nvlink_bytes=886164720 weight_to_token_ratio=0.594213 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057618584e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280125440 dispatch_meta_bytes=229536 combine_payload_bytes=280125440 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=280354976 ingress_bytes=390225920 link_bytes=390225920 token_bytes=560480416 weight_movement_bytes=110100480 total_nvlink_bytes=670580896 weight_to_token_ratio=0.196439 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786670560e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=39080 dup_rows=20940 dup_rows_remote=18304 dedup_ratio=0.319519 router_tokens_per_expert=[527, 520, 533, 524, 518, 494, 534, 514, 486, 538, 551, 473, 496, 508, 533, 498, 485, 501, 524, 508, 485, 489, 519, 497, 518, 496, 521, 474, 501, 509, 529, 538, 545, 506, 511, 494, 536, 510, 499, 555, 524, 518, 520, 523, 498, 516, 485, 512, 509, 502, 534, 493, 455, 515, 553, 515, 499, 521, 527, 542, 522, 490, 523, 475] payload_rows_per_dst=[5604, 5612, 5472, 5594, 5614, 5612, 5516, 5572] dup_rows_per_dst=[2724, 2554, 2544, 2578, 2698, 2580, 2636, 2626] combine_rows_per_src=[5604, 5612, 5472, 5594, 5614, 5612, 5516, 5572] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277473280 dispatch_meta_bytes=227696 combine_payload_bytes=277473280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277700976 ingress_bytes=387573760 link_bytes=387573760 token_bytes=555174256 weight_movement_bytes=110100480 total_nvlink_bytes=665274736 weight_to_token_ratio=0.198317 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.740545136e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277745664 dispatch_meta_bytes=229360 combine_payload_bytes=277745664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277975024 ingress_bytes=497946624 link_bytes=497946624 token_bytes=555720688 weight_movement_bytes=220200960 total_nvlink_bytes=775921648 weight_to_token_ratio=0.396244 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.660110773e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277817344 dispatch_meta_bytes=228592 combine_payload_bytes=277817344 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278045936 ingress_bytes=608118784 link_bytes=608118784 token_bytes=555863280 weight_movement_bytes=330301440 total_nvlink_bytes=886164720 weight_to_token_ratio=0.594213 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057618584e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=229496 combine_payload_bytes=279652352 prefetch_bytes=0 reduce_bytes=0 egress_bytes=279881848 ingress_bytes=279652352 link_bytes=279881848 token_bytes=559534200 weight_movement_bytes=0 total_nvlink_bytes=559534200 weight_to_token_ratio=0.000000 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=4.867605663e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44706 payload_rows_remote=39156 dup_rows=20830 dup_rows_remote=18262 dedup_ratio=0.317841 router_tokens_per_expert=[522, 493, 494, 519, 548, 493, 511, 471, 513, 517, 520, 519, 483, 523, 500, 517, 523, 510, 515, 500, 565, 513, 504, 504, 490, 524, 513, 519, 486, 523, 498, 506, 520, 502, 529, 524, 510, 511, 478, 500, 481, 537, 509, 504, 514, 509, 516, 517, 522, 481, 543, 525, 503, 504, 542, 497, 510, 533, 527, 530, 552, 537, 482, 483] payload_rows_per_dst=[5576, 5576, 5616, 5550, 5558, 5630, 5588, 5612] dup_rows_per_dst=[2526, 2608, 2652, 2568, 2590, 2544, 2646, 2696] combine_rows_per_src=[5576, 5576, 5616, 5550, 5558, 5630, 5588, 5612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=228920 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279436856 ingress_bytes=609509376 link_bytes=609509376 token_bytes=558644792 weight_movement_bytes=330301440 total_nvlink_bytes=888946232 weight_to_token_ratio=0.591255 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060037052e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=38938 dup_rows=20950 dup_rows_remote=18322 dedup_ratio=0.319672 router_tokens_per_expert=[517, 520, 540, 501, 487, 444, 526, 545, 525, 504, 535, 518, 504, 490, 496, 495, 510, 515, 513, 515, 540, 464, 522, 559, 469, 518, 473, 549, 539, 500, 473, 520, 487, 528, 530, 503, 538, 509, 521, 482, 490, 521, 514, 534, 568, 546, 493, 530, 510, 526, 505, 519, 534, 484, 513, 504, 459, 507, 518, 498, 528, 510, 525, 508] payload_rows_per_dst=[5574, 5558, 5648, 5510, 5500, 5644, 5574, 5578] dup_rows_per_dst=[2586, 2576, 2628, 2572, 2696, 2748, 2616, 2528] combine_rows_per_src=[5574, 5558, 5648, 5510, 5500, 5644, 5574, 5578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280125440 dispatch_meta_bytes=229536 combine_payload_bytes=280125440 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=280354976 ingress_bytes=390225920 link_bytes=390225920 token_bytes=560480416 weight_movement_bytes=110100480 total_nvlink_bytes=670580896 weight_to_token_ratio=0.196439 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786670560e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280670208 dispatch_meta_bytes=229672 combine_payload_bytes=280670208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280899880 ingress_bytes=390770688 link_bytes=390770688 token_bytes=561570088 weight_movement_bytes=110100480 total_nvlink_bytes=671670568 weight_to_token_ratio=0.196058 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.796144972e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229040 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279336624 ingress_bytes=389208064 link_bytes=389208064 token_bytes=558444208 weight_movement_bytes=110100480 total_nvlink_bytes=668544688 weight_to_token_ratio=0.197156 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768968370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38730 dup_rows=21118 dup_rows_remote=18436 dedup_ratio=0.322235 router_tokens_per_expert=[510, 490, 557, 540, 526, 488, 493, 479, 517, 513, 483, 483, 515, 520, 491, 516, 538, 497, 517, 490, 503, 529, 546, 510, 519, 491, 546, 518, 539, 524, 511, 502, 530, 509, 486, 530, 511, 497, 516, 507, 515, 472, 559, 527, 495, 508, 509, 489, 476, 531, 488, 502, 467, 505, 543, 526, 552, 463, 541, 498, 523, 535, 489, 568] payload_rows_per_dst=[5568, 5538, 5450, 5432, 5486, 5502, 5754, 5688] dup_rows_per_dst=[2598, 2538, 2472, 2580, 2654, 2646, 2948, 2682] combine_rows_per_src=[5568, 5538, 5450, 5432, 5486, 5502, 5754, 5688] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38932 dup_rows=21058 dup_rows_remote=18486 dedup_ratio=0.321320 router_tokens_per_expert=[492, 475, 534, 494, 503, 521, 530, 494, 538, 541, 481, 507, 490, 475, 493, 503, 499, 492, 506, 517, 540, 494, 499, 551, 504, 483, 523, 539, 524, 510, 537, 555, 499, 510, 479, 501, 505, 478, 532, 555, 514, 506, 500, 535, 499, 453, 494, 514, 501, 534, 507, 490, 543, 532, 525, 536, 464, 536, 501, 544, 566, 548, 511, 512] payload_rows_per_dst=[5498, 5490, 5544, 5632, 5546, 5458, 5638, 5672] dup_rows_per_dst=[2588, 2566, 2652, 2718, 2572, 2572, 2698, 2692] combine_rows_per_src=[5498, 5490, 5544, 5632, 5546, 5458, 5638, 5672] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44404 payload_rows_remote=38934 dup_rows=21132 dup_rows_remote=18636 dedup_ratio=0.322449 router_tokens_per_expert=[507, 501, 495, 488, 514, 519, 503, 546, 479, 514, 495, 495, 497, 516, 487, 500, 486, 503, 501, 540, 528, 535, 533, 521, 512, 498, 533, 500, 509, 515, 538, 495, 525, 525, 517, 532, 551, 520, 496, 518, 503, 503, 493, 522, 550, 518, 484, 506, 544, 458, 524, 527, 493, 520, 481, 491, 530, 513, 495, 532, 514, 520, 510, 550] payload_rows_per_dst=[5580, 5470, 5570, 5618, 5590, 5474, 5480, 5622] dup_rows_per_dst=[2566, 2496, 2724, 2582, 2778, 2684, 2596, 2706] combine_rows_per_src=[5580, 5470, 5570, 5618, 5590, 5474, 5480, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=229688 combine_payload_bytes=279336960 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279566648 ingress_bytes=389437440 link_bytes=389437440 token_bytes=558903608 weight_movement_bytes=110100480 total_nvlink_bytes=669004088 weight_to_token_ratio=0.196994 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772957596e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=228920 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279436856 ingress_bytes=609509376 link_bytes=609509376 token_bytes=558644792 weight_movement_bytes=330301440 total_nvlink_bytes=888946232 weight_to_token_ratio=0.591255 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060037052e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=229496 combine_payload_bytes=279652352 prefetch_bytes=0 reduce_bytes=0 egress_bytes=279881848 ingress_bytes=279652352 link_bytes=279881848 token_bytes=559534200 weight_movement_bytes=0 total_nvlink_bytes=559534200 weight_to_token_ratio=0.000000 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=4.867605663e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=38932 dup_rows=20990 dup_rows_remote=18254 dedup_ratio=0.320282 router_tokens_per_expert=[509, 515, 524, 523, 508, 498, 506, 527, 540, 518, 542, 505, 518, 499, 536, 473, 490, 512, 539, 516, 507, 521, 550, 540, 512, 503, 472, 507, 533, 552, 527, 499, 528, 482, 496, 535, 529, 504, 523, 473, 479, 508, 516, 490, 536, 505, 514, 540, 517, 508, 491, 483, 485, 514, 522, 507, 511, 538, 524, 527, 511, 495, 481, 475] payload_rows_per_dst=[5618, 5534, 5614, 5606, 5546, 5654, 5486, 5488] dup_rows_per_dst=[2602, 2728, 2736, 2604, 2594, 2522, 2568, 2636] combine_rows_per_src=[5618, 5534, 5614, 5606, 5546, 5654, 5486, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38710 dup_rows=21142 dup_rows_remote=18214 dedup_ratio=0.322601 router_tokens_per_expert=[511, 523, 539, 520, 468, 529, 557, 539, 525, 478, 539, 500, 511, 487, 550, 511, 512, 508, 508, 519, 506, 469, 517, 528, 542, 486, 487, 505, 509, 522, 490, 506, 499, 542, 518, 505, 527, 529, 482, 533, 504, 480, 499, 517, 526, 483, 505, 500, 496, 536, 507, 504, 498, 531, 525, 496, 516, 478, 519, 519, 518, 525, 531, 519] payload_rows_per_dst=[5516, 5592, 5580, 5594, 5538, 5330, 5560, 5684] dup_rows_per_dst=[2552, 2610, 2554, 2748, 2732, 2392, 2626, 2928] combine_rows_per_src=[5516, 5592, 5580, 5594, 5538, 5330, 5560, 5684] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44258 payload_rows_remote=38576 dup_rows=21278 dup_rows_remote=18434 dedup_ratio=0.324677 router_tokens_per_expert=[513, 513, 495, 496, 562, 527, 513, 565, 496, 524, 504, 483, 503, 504, 462, 507, 555, 518, 500, 485, 492, 485, 516, 539, 479, 471, 538, 532, 542, 504, 511, 502, 493, 502, 505, 555, 513, 532, 550, 509, 522, 485, 529, 506, 486, 520, 533, 500, 509, 546, 515, 516, 518, 513, 522, 560, 519, 500, 502, 477, 473, 510, 485, 527] payload_rows_per_dst=[5682, 5738, 5592, 5496, 5582, 5570, 5212, 5386] dup_rows_per_dst=[2844, 3166, 2660, 2662, 2578, 2740, 2248, 2380] combine_rows_per_src=[5682, 5738, 5592, 5496, 5582, 5570, 5212, 5386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=38936 dup_rows=21044 dup_rows_remote=18430 dedup_ratio=0.321106 router_tokens_per_expert=[516, 497, 494, 535, 532, 507, 505, 474, 484, 527, 509, 522, 491, 518, 445, 517, 557, 535, 530, 520, 535, 542, 517, 504, 535, 489, 514, 526, 518, 511, 538, 515, 529, 567, 548, 493, 520, 496, 507, 495, 509, 508, 523, 508, 541, 483, 522, 491, 487, 533, 496, 517, 513, 494, 486, 510, 518, 540, 492, 519, 514, 510, 454, 486] payload_rows_per_dst=[5536, 5506, 5658, 5616, 5568, 5556, 5492, 5560] dup_rows_per_dst=[2584, 2520, 2822, 2676, 2742, 2614, 2580, 2506] combine_rows_per_src=[5536, 5506, 5658, 5616, 5568, 5556, 5492, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=228744 combine_payload_bytes=279064576 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=279293320 ingress_bytes=829566976 link_bytes=829566976 token_bytes=558357896 weight_movement_bytes=550502400 total_nvlink_bytes=1108860296 weight_to_token_ratio=0.985931 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.442753412e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44462 payload_rows_remote=38882 dup_rows=21074 dup_rows_remote=18374 dedup_ratio=0.321564 router_tokens_per_expert=[499, 541, 478, 485, 505, 527, 482, 477, 509, 511, 509, 497, 530, 528, 513, 523, 453, 500, 512, 560, 532, 474, 538, 483, 522, 528, 482, 528, 523, 526, 451, 500, 545, 485, 504, 535, 517, 489, 524, 500, 514, 563, 540, 550, 495, 491, 524, 507, 536, 523, 523, 525, 477, 528, 514, 514, 532, 504, 461, 499, 530, 545, 523, 525] payload_rows_per_dst=[5440, 5524, 5546, 5600, 5540, 5638, 5580, 5594] dup_rows_per_dst=[2548, 2716, 2558, 2520, 2658, 2730, 2700, 2644] combine_rows_per_src=[5440, 5524, 5546, 5600, 5540, 5638, 5580, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277616640 dispatch_meta_bytes=228664 combine_payload_bytes=277616640 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277845304 ingress_bytes=607918080 link_bytes=607918080 token_bytes=555461944 weight_movement_bytes=330301440 total_nvlink_bytes=885763384 weight_to_token_ratio=0.594643 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057269526e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=229672 combine_payload_bytes=279064576 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279294248 ingress_bytes=499265536 link_bytes=499265536 token_bytes=558358824 weight_movement_bytes=220200960 total_nvlink_bytes=778559784 weight_to_token_ratio=0.394372 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683048822e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=230280 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279309192 ingress_bytes=389179392 link_bytes=389179392 token_bytes=558388104 weight_movement_bytes=110100480 total_nvlink_bytes=668488584 weight_to_token_ratio=0.197176 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768469717e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276512768 dispatch_meta_bytes=228040 combine_payload_bytes=276512768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276740808 ingress_bytes=386613248 link_bytes=386613248 token_bytes=553253576 weight_movement_bytes=110100480 total_nvlink_bytes=663354056 weight_to_token_ratio=0.199005 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.723840252e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279093248 dispatch_meta_bytes=229464 combine_payload_bytes=279093248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279322712 ingress_bytes=389193728 link_bytes=389193728 token_bytes=558415960 weight_movement_bytes=110100480 total_nvlink_bytes=668516440 weight_to_token_ratio=0.197166 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768719044e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39034 dup_rows=20968 dup_rows_remote=18300 dedup_ratio=0.319946 router_tokens_per_expert=[509, 533, 525, 515, 530, 501, 538, 495, 524, 520, 485, 515, 523, 527, 490, 494, 524, 496, 500, 494, 522, 508, 506, 503, 510, 521, 485, 496, 514, 544, 513, 518, 501, 525, 499, 525, 528, 489, 515, 503, 513, 506, 487, 516, 510, 500, 551, 534, 492, 517, 536, 512, 512, 503, 552, 518, 515, 478, 524, 536, 478, 483, 489, 543] payload_rows_per_dst=[5608, 5536, 5582, 5534, 5580, 5604, 5622, 5502] dup_rows_per_dst=[2684, 2620, 2524, 2668, 2590, 2630, 2662, 2590] combine_rows_per_src=[5608, 5536, 5582, 5534, 5580, 5604, 5622, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277473280 dispatch_meta_bytes=227696 combine_payload_bytes=277473280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277700976 ingress_bytes=387573760 link_bytes=387573760 token_bytes=555174256 weight_movement_bytes=110100480 total_nvlink_bytes=665274736 weight_to_token_ratio=0.198317 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.740545136e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44352 payload_rows_remote=38752 dup_rows=21184 dup_rows_remote=18602 dedup_ratio=0.323242 router_tokens_per_expert=[530, 532, 533, 517, 514, 534, 470, 476, 533, 490, 510, 481, 474, 532, 514, 522, 522, 509, 510, 486, 487, 506, 526, 545, 512, 501, 523, 540, 485, 503, 526, 511, 497, 523, 501, 490, 514, 491, 496, 524, 496, 509, 506, 513, 493, 558, 506, 478, 532, 506, 544, 501, 525, 514, 524, 517, 535, 553, 523, 516, 470, 514, 531, 514] payload_rows_per_dst=[5546, 5552, 5600, 5526, 5500, 5456, 5596, 5576] dup_rows_per_dst=[2666, 2560, 2582, 2676, 2572, 2662, 2730, 2736] combine_rows_per_src=[5546, 5552, 5600, 5526, 5500, 5456, 5596, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44356 payload_rows_remote=38748 dup_rows=21180 dup_rows_remote=18592 dedup_ratio=0.323181 router_tokens_per_expert=[496, 497, 532, 508, 501, 503, 502, 527, 555, 523, 485, 475, 518, 487, 556, 499, 503, 527, 519, 522, 512, 499, 504, 496, 506, 527, 512, 515, 547, 511, 495, 537, 491, 529, 502, 505, 546, 523, 451, 554, 547, 504, 529, 534, 524, 480, 504, 540, 501, 537, 498, 524, 494, 497, 554, 500, 489, 517, 517, 485, 529, 494, 487, 486] payload_rows_per_dst=[5538, 5608, 5542, 5598, 5494, 5574, 5576, 5426] dup_rows_per_dst=[2594, 2588, 2622, 2702, 2708, 2750, 2634, 2582] combine_rows_per_src=[5538, 5608, 5542, 5598, 5494, 5574, 5576, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276512768 dispatch_meta_bytes=228040 combine_payload_bytes=276512768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276740808 ingress_bytes=386613248 link_bytes=386613248 token_bytes=553253576 weight_movement_bytes=110100480 total_nvlink_bytes=663354056 weight_to_token_ratio=0.199005 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.723840252e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=38936 dup_rows=21044 dup_rows_remote=18430 dedup_ratio=0.321106 router_tokens_per_expert=[516, 497, 494, 535, 532, 507, 505, 474, 484, 527, 509, 522, 491, 518, 445, 517, 557, 535, 530, 520, 535, 542, 517, 504, 535, 489, 514, 526, 518, 511, 538, 515, 529, 567, 548, 493, 520, 496, 507, 495, 509, 508, 523, 508, 541, 483, 522, 491, 487, 533, 496, 517, 513, 494, 486, 510, 518, 540, 492, 519, 514, 510, 454, 486] payload_rows_per_dst=[5536, 5506, 5658, 5616, 5568, 5556, 5492, 5560] dup_rows_per_dst=[2584, 2520, 2822, 2676, 2742, 2614, 2580, 2506] combine_rows_per_src=[5536, 5506, 5658, 5616, 5568, 5556, 5492, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38932 dup_rows=21058 dup_rows_remote=18486 dedup_ratio=0.321320 router_tokens_per_expert=[492, 475, 534, 494, 503, 521, 530, 494, 538, 541, 481, 507, 490, 475, 493, 503, 499, 492, 506, 517, 540, 494, 499, 551, 504, 483, 523, 539, 524, 510, 537, 555, 499, 510, 479, 501, 505, 478, 532, 555, 514, 506, 500, 535, 499, 453, 494, 514, 501, 534, 507, 490, 543, 532, 525, 536, 464, 536, 501, 544, 566, 548, 511, 512] payload_rows_per_dst=[5498, 5490, 5544, 5632, 5546, 5458, 5638, 5672] dup_rows_per_dst=[2588, 2566, 2652, 2718, 2572, 2572, 2698, 2692] combine_rows_per_src=[5498, 5490, 5544, 5632, 5546, 5458, 5638, 5672] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=229024 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278935200 ingress_bytes=498907136 link_bytes=498907136 token_bytes=557641376 weight_movement_bytes=220200960 total_nvlink_bytes=777842336 weight_to_token_ratio=0.394879 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676815656e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229336 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280025048 ingress_bytes=499996672 link_bytes=499996672 token_bytes=559820760 weight_movement_bytes=220200960 total_nvlink_bytes=780021720 weight_to_token_ratio=0.393342 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.695764479e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277774336 dispatch_meta_bytes=229416 combine_payload_bytes=277774336 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278003752 ingress_bytes=497975296 link_bytes=497975296 token_bytes=555778088 weight_movement_bytes=220200960 total_nvlink_bytes=775979048 weight_to_token_ratio=0.396203 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.660609426e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=38874 dup_rows=21044 dup_rows_remote=18316 dedup_ratio=0.321106 router_tokens_per_expert=[539, 513, 507, 509, 502, 531, 496, 506, 512, 518, 501, 492, 504, 501, 500, 494, 512, 529, 478, 521, 553, 484, 515, 502, 491, 539, 482, 494, 544, 483, 540, 531, 530, 512, 520, 498, 502, 535, 542, 534, 524, 512, 536, 484, 507, 523, 502, 463, 520, 531, 498, 518, 529, 511, 533, 526, 483, 512, 515, 505, 497, 519, 512, 512] payload_rows_per_dst=[5626, 5556, 5578, 5524, 5618, 5486, 5570, 5534] dup_rows_per_dst=[2580, 2488, 2610, 2684, 2728, 2616, 2762, 2576] combine_rows_per_src=[5626, 5556, 5578, 5524, 5618, 5486, 5570, 5534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=39006 dup_rows=20940 dup_rows_remote=18266 dedup_ratio=0.319519 router_tokens_per_expert=[456, 523, 519, 515, 520, 544, 513, 490, 504, 531, 513, 521, 518, 513, 549, 510, 533, 511, 489, 492, 512, 494, 482, 514, 488, 544, 518, 519, 529, 532, 522, 505, 520, 553, 529, 534, 515, 492, 527, 488, 494, 504, 487, 490, 536, 517, 498, 455, 545, 507, 483, 482, 539, 480, 536, 508, 497, 551, 492, 548, 540, 464, 525, 509] payload_rows_per_dst=[5476, 5764, 5512, 5582, 5620, 5502, 5550, 5590] dup_rows_per_dst=[2372, 2866, 2484, 2732, 2754, 2460, 2598, 2674] combine_rows_per_src=[5476, 5764, 5512, 5582, 5620, 5502, 5550, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38884 dup_rows=21084 dup_rows_remote=18448 dedup_ratio=0.321716 router_tokens_per_expert=[522, 502, 507, 476, 515, 487, 515, 550, 514, 511, 499, 530, 494, 536, 502, 516, 487, 491, 534, 526, 455, 527, 515, 511, 472, 503, 532, 508, 522, 548, 521, 506, 510, 531, 569, 517, 518, 500, 482, 524, 467, 553, 520, 512, 479, 557, 540, 536, 493, 481, 535, 472, 495, 468, 515, 510, 491, 502, 492, 526, 536, 529, 539, 535] payload_rows_per_dst=[5576, 5568, 5488, 5528, 5630, 5622, 5500, 5540] dup_rows_per_dst=[2572, 2636, 2604, 2696, 2672, 2706, 2438, 2760] combine_rows_per_src=[5576, 5568, 5488, 5528, 5630, 5622, 5500, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38946 dup_rows=21116 dup_rows_remote=18604 dedup_ratio=0.322205 router_tokens_per_expert=[492, 528, 517, 462, 512, 549, 520, 488, 508, 542, 504, 508, 500, 499, 487, 531, 550, 497, 493, 516, 549, 511, 524, 528, 484, 527, 494, 486, 511, 500, 493, 536, 505, 497, 518, 503, 539, 480, 523, 522, 512, 514, 503, 493, 477, 512, 493, 489, 489, 505, 529, 530, 499, 482, 498, 535, 493, 536, 524, 556, 556, 571, 520, 519] payload_rows_per_dst=[5498, 5488, 5572, 5564, 5564, 5474, 5574, 5686] dup_rows_per_dst=[2638, 2670, 2764, 2498, 2610, 2512, 2560, 2864] combine_rows_per_src=[5498, 5488, 5572, 5564, 5564, 5474, 5574, 5686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279093248 dispatch_meta_bytes=229464 combine_payload_bytes=279093248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279322712 ingress_bytes=389193728 link_bytes=389193728 token_bytes=558415960 weight_movement_bytes=110100480 total_nvlink_bytes=668516440 weight_to_token_ratio=0.197166 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768719044e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277745664 dispatch_meta_bytes=229360 combine_payload_bytes=277745664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277975024 ingress_bytes=497946624 link_bytes=497946624 token_bytes=555720688 weight_movement_bytes=220200960 total_nvlink_bytes=775921648 weight_to_token_ratio=0.396244 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.660110773e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=229672 combine_payload_bytes=279064576 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279294248 ingress_bytes=499265536 link_bytes=499265536 token_bytes=558358824 weight_movement_bytes=220200960 total_nvlink_bytes=778559784 weight_to_token_ratio=0.394372 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683048822e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=38938 dup_rows=20950 dup_rows_remote=18322 dedup_ratio=0.319672 router_tokens_per_expert=[517, 520, 540, 501, 487, 444, 526, 545, 525, 504, 535, 518, 504, 490, 496, 495, 510, 515, 513, 515, 540, 464, 522, 559, 469, 518, 473, 549, 539, 500, 473, 520, 487, 528, 530, 503, 538, 509, 521, 482, 490, 521, 514, 534, 568, 546, 493, 530, 510, 526, 505, 519, 534, 484, 513, 504, 459, 507, 518, 498, 528, 510, 525, 508] payload_rows_per_dst=[5574, 5558, 5648, 5510, 5500, 5644, 5574, 5578] dup_rows_per_dst=[2586, 2576, 2628, 2572, 2696, 2748, 2616, 2528] combine_rows_per_src=[5574, 5558, 5648, 5510, 5500, 5644, 5574, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38730 dup_rows=21118 dup_rows_remote=18436 dedup_ratio=0.322235 router_tokens_per_expert=[510, 490, 557, 540, 526, 488, 493, 479, 517, 513, 483, 483, 515, 520, 491, 516, 538, 497, 517, 490, 503, 529, 546, 510, 519, 491, 546, 518, 539, 524, 511, 502, 530, 509, 486, 530, 511, 497, 516, 507, 515, 472, 559, 527, 495, 508, 509, 489, 476, 531, 488, 502, 467, 505, 543, 526, 552, 463, 541, 498, 523, 535, 489, 568] payload_rows_per_dst=[5568, 5538, 5450, 5432, 5486, 5502, 5754, 5688] dup_rows_per_dst=[2598, 2538, 2472, 2580, 2654, 2646, 2948, 2682] combine_rows_per_src=[5568, 5538, 5450, 5432, 5486, 5502, 5754, 5688] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38746 dup_rows=21086 dup_rows_remote=18398 dedup_ratio=0.321747 router_tokens_per_expert=[525, 525, 516, 505, 509, 506, 532, 577, 538, 495, 526, 473, 522, 503, 511, 520, 535, 499, 505, 538, 525, 488, 495, 515, 503, 503, 545, 495, 512, 500, 477, 504, 501, 496, 515, 500, 516, 491, 477, 497, 504, 452, 500, 526, 515, 516, 534, 500, 513, 529, 507, 556, 517, 495, 524, 556, 521, 534, 504, 503, 500, 513, 537, 497] payload_rows_per_dst=[5704, 5530, 5588, 5512, 5434, 5594, 5634, 5454] dup_rows_per_dst=[2688, 2646, 2642, 2684, 2552, 2626, 2758, 2490] combine_rows_per_src=[5704, 5530, 5588, 5512, 5434, 5594, 5634, 5454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44434 payload_rows_remote=38828 dup_rows=21102 dup_rows_remote=18408 dedup_ratio=0.321991 router_tokens_per_expert=[546, 463, 525, 513, 528, 516, 542, 522, 495, 477, 510, 485, 498, 497, 511, 500, 532, 515, 518, 499, 462, 514, 550, 503, 497, 510, 529, 497, 511, 536, 537, 494, 523, 510, 502, 499, 518, 503, 518, 501, 516, 487, 550, 506, 532, 544, 550, 542, 510, 514, 560, 531, 508, 514, 505, 508, 497, 483, 519, 480, 498, 482, 528, 498] payload_rows_per_dst=[5620, 5414, 5552, 5582, 5490, 5692, 5606, 5478] dup_rows_per_dst=[2690, 2532, 2634, 2640, 2658, 2762, 2694, 2492] combine_rows_per_src=[5620, 5414, 5552, 5582, 5490, 5692, 5606, 5478] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=228760 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278877592 ingress_bytes=388749312 link_bytes=388749312 token_bytes=557526424 weight_movement_bytes=110100480 total_nvlink_bytes=667626904 weight_to_token_ratio=0.197480 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760989918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=229088 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279824096 ingress_bytes=719996928 link_bytes=719996928 token_bytes=559419104 weight_movement_bytes=440401920 total_nvlink_bytes=999821024 weight_to_token_ratio=0.787249 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252193077e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229328 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278949840 ingress_bytes=388820992 link_bytes=388820992 token_bytes=557670352 weight_movement_bytes=110100480 total_nvlink_bytes=667770832 weight_to_token_ratio=0.197429 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762236552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=230200 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279395128 ingress_bytes=389265408 link_bytes=389265408 token_bytes=558560056 weight_movement_bytes=110100480 total_nvlink_bytes=668660536 weight_to_token_ratio=0.197115 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769965677e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277731328 dispatch_meta_bytes=228576 combine_payload_bytes=277731328 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277959904 ingress_bytes=497932288 link_bytes=497932288 token_bytes=555691232 weight_movement_bytes=220200960 total_nvlink_bytes=775892192 weight_to_token_ratio=0.396265 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.659861446e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229040 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279336624 ingress_bytes=389208064 link_bytes=389208064 token_bytes=558444208 weight_movement_bytes=110100480 total_nvlink_bytes=668544688 weight_to_token_ratio=0.197156 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768968370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277616640 dispatch_meta_bytes=228664 combine_payload_bytes=277616640 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277845304 ingress_bytes=607918080 link_bytes=607918080 token_bytes=555461944 weight_movement_bytes=330301440 total_nvlink_bytes=885763384 weight_to_token_ratio=0.594643 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057269526e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=39066 dup_rows=20948 dup_rows_remote=18338 dedup_ratio=0.319641 router_tokens_per_expert=[463, 490, 504, 490, 544, 518, 552, 521, 537, 507, 487, 501, 528, 538, 482, 541, 537, 514, 496, 531, 481, 517, 485, 525, 531, 519, 506, 514, 512, 475, 520, 489, 510, 491, 523, 496, 541, 522, 501, 513, 510, 498, 538, 506, 480, 522, 486, 531, 487, 529, 532, 490, 516, 539, 488, 513, 510, 506, 530, 528, 500, 512, 519, 546] payload_rows_per_dst=[5600, 5564, 5582, 5522, 5594, 5544, 5530, 5652] dup_rows_per_dst=[2564, 2678, 2590, 2610, 2600, 2598, 2658, 2650] combine_rows_per_src=[5600, 5564, 5582, 5522, 5594, 5544, 5530, 5652] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44550 payload_rows_remote=38896 dup_rows=20986 dup_rows_remote=18304 dedup_ratio=0.320221 router_tokens_per_expert=[513, 532, 521, 527, 555, 513, 502, 503, 522, 490, 521, 497, 508, 517, 494, 519, 537, 533, 541, 511, 495, 513, 534, 504, 493, 504, 518, 515, 528, 475, 523, 533, 509, 422, 500, 502, 511, 491, 555, 516, 497, 562, 489, 529, 534, 499, 531, 530, 491, 469, 522, 545, 535, 499, 499, 492, 508, 497, 522, 482, 493, 526, 513, 507] payload_rows_per_dst=[5598, 5534, 5654, 5572, 5504, 5652, 5488, 5548] dup_rows_per_dst=[2734, 2602, 2682, 2606, 2508, 2690, 2616, 2548] combine_rows_per_src=[5598, 5534, 5654, 5572, 5504, 5652, 5488, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277731328 dispatch_meta_bytes=228576 combine_payload_bytes=277731328 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277959904 ingress_bytes=497932288 link_bytes=497932288 token_bytes=555691232 weight_movement_bytes=220200960 total_nvlink_bytes=775892192 weight_to_token_ratio=0.396265 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.659861446e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38946 dup_rows=21116 dup_rows_remote=18604 dedup_ratio=0.322205 router_tokens_per_expert=[492, 528, 517, 462, 512, 549, 520, 488, 508, 542, 504, 508, 500, 499, 487, 531, 550, 497, 493, 516, 549, 511, 524, 528, 484, 527, 494, 486, 511, 500, 493, 536, 505, 497, 518, 503, 539, 480, 523, 522, 512, 514, 503, 493, 477, 512, 493, 489, 489, 505, 529, 530, 499, 482, 498, 535, 493, 536, 524, 556, 556, 571, 520, 519] payload_rows_per_dst=[5498, 5488, 5572, 5564, 5564, 5474, 5574, 5686] dup_rows_per_dst=[2638, 2670, 2764, 2498, 2610, 2512, 2560, 2864] combine_rows_per_src=[5498, 5488, 5572, 5564, 5564, 5474, 5574, 5686] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44404 payload_rows_remote=38934 dup_rows=21132 dup_rows_remote=18636 dedup_ratio=0.322449 router_tokens_per_expert=[507, 501, 495, 488, 514, 519, 503, 546, 479, 514, 495, 495, 497, 516, 487, 500, 486, 503, 501, 540, 528, 535, 533, 521, 512, 498, 533, 500, 509, 515, 538, 495, 525, 525, 517, 532, 551, 520, 496, 518, 503, 503, 493, 522, 550, 518, 484, 506, 544, 458, 524, 527, 493, 520, 481, 491, 530, 513, 495, 532, 514, 520, 510, 550] payload_rows_per_dst=[5580, 5470, 5570, 5618, 5590, 5474, 5480, 5622] dup_rows_per_dst=[2566, 2496, 2724, 2582, 2778, 2684, 2596, 2706] combine_rows_per_src=[5580, 5470, 5570, 5618, 5590, 5474, 5480, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=38874 dup_rows=21044 dup_rows_remote=18316 dedup_ratio=0.321106 router_tokens_per_expert=[539, 513, 507, 509, 502, 531, 496, 506, 512, 518, 501, 492, 504, 501, 500, 494, 512, 529, 478, 521, 553, 484, 515, 502, 491, 539, 482, 494, 544, 483, 540, 531, 530, 512, 520, 498, 502, 535, 542, 534, 524, 512, 536, 484, 507, 523, 502, 463, 520, 531, 498, 518, 529, 511, 533, 526, 483, 512, 515, 505, 497, 519, 512, 512] payload_rows_per_dst=[5626, 5556, 5578, 5524, 5618, 5486, 5570, 5534] dup_rows_per_dst=[2580, 2488, 2610, 2684, 2728, 2616, 2762, 2576] combine_rows_per_src=[5626, 5556, 5578, 5524, 5618, 5486, 5570, 5534] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278319104 dispatch_meta_bytes=228944 combine_payload_bytes=278319104 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278548048 ingress_bytes=498520064 link_bytes=498520064 token_bytes=556867152 weight_movement_bytes=220200960 total_nvlink_bytes=777068112 weight_to_token_ratio=0.395428 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670083838e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=229616 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280254704 ingress_bytes=390125568 link_bytes=390125568 token_bytes=560279792 weight_movement_bytes=110100480 total_nvlink_bytes=670380272 weight_to_token_ratio=0.196510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784925274e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=228800 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279035328 ingress_bytes=499007488 link_bytes=499007488 token_bytes=557841856 weight_movement_bytes=220200960 total_nvlink_bytes=778042816 weight_to_token_ratio=0.394737 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678560943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38840 dup_rows=21158 dup_rows_remote=18462 dedup_ratio=0.322845 router_tokens_per_expert=[509, 533, 488, 493, 528, 492, 510, 524, 544, 529, 515, 513, 501, 520, 487, 546, 547, 483, 537, 494, 502, 529, 510, 457, 520, 510, 467, 493, 537, 515, 489, 558, 497, 516, 513, 522, 542, 531, 505, 491, 516, 499, 528, 471, 515, 507, 488, 514, 503, 516, 548, 549, 516, 500, 497, 531, 544, 498, 519, 509, 490, 514, 488, 511] payload_rows_per_dst=[5486, 5558, 5552, 5588, 5538, 5486, 5626, 5544] dup_rows_per_dst=[2668, 2752, 2566, 2590, 2696, 2590, 2694, 2602] combine_rows_per_src=[5486, 5558, 5552, 5588, 5538, 5486, 5626, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44212 payload_rows_remote=38698 dup_rows=21324 dup_rows_remote=18558 dedup_ratio=0.325378 router_tokens_per_expert=[511, 525, 500, 568, 464, 507, 530, 538, 517, 522, 505, 513, 474, 504, 499, 519, 485, 505, 545, 536, 502, 478, 494, 502, 502, 514, 478, 510, 490, 560, 525, 540, 524, 533, 500, 524, 537, 477, 523, 534, 535, 512, 527, 513, 487, 493, 504, 505, 485, 506, 516, 511, 517, 472, 520, 528, 509, 465, 515, 529, 520, 524, 528, 533] payload_rows_per_dst=[5342, 5586, 5630, 5552, 5572, 5518, 5498, 5514] dup_rows_per_dst=[2542, 2520, 2866, 2686, 2698, 2634, 2612, 2766] combine_rows_per_src=[5342, 5586, 5630, 5552, 5572, 5518, 5498, 5514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44642 payload_rows_remote=39060 dup_rows=20894 dup_rows_remote=18328 dedup_ratio=0.318817 router_tokens_per_expert=[514, 522, 529, 501, 494, 541, 488, 510, 534, 507, 494, 521, 501, 514, 493, 510, 483, 543, 530, 489, 517, 509, 529, 476, 517, 536, 504, 497, 517, 502, 560, 514, 538, 483, 488, 481, 477, 515, 549, 499, 522, 545, 528, 513, 516, 510, 508, 506, 449, 527, 503, 524, 513, 508, 507, 557, 498, 531, 519, 530, 508, 525, 484, 511] payload_rows_per_dst=[5622, 5582, 5612, 5610, 5570, 5608, 5540, 5498] dup_rows_per_dst=[2576, 2566, 2540, 2684, 2490, 2688, 2636, 2714] combine_rows_per_src=[5622, 5582, 5612, 5610, 5570, 5608, 5540, 5498] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44262 payload_rows_remote=38734 dup_rows=21274 dup_rows_remote=18564 dedup_ratio=0.324615 router_tokens_per_expert=[496, 512, 516, 531, 476, 486, 523, 483, 485, 503, 478, 538, 505, 515, 475, 524, 515, 514, 513, 529, 573, 484, 529, 520, 510, 513, 504, 540, 511, 500, 488, 496, 521, 500, 506, 524, 539, 525, 509, 502, 500, 464, 503, 521, 546, 510, 543, 532, 502, 526, 529, 513, 501, 520, 555, 495, 520, 488, 520, 531, 523, 482, 520, 513] payload_rows_per_dst=[5476, 5512, 5518, 5508, 5566, 5528, 5618, 5536] dup_rows_per_dst=[2570, 2534, 2836, 2616, 2686, 2710, 2664, 2658] combine_rows_per_src=[5476, 5512, 5518, 5508, 5566, 5528, 5618, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=230200 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279395128 ingress_bytes=389265408 link_bytes=389265408 token_bytes=558560056 weight_movement_bytes=110100480 total_nvlink_bytes=668660536 weight_to_token_ratio=0.197115 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769965677e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=230280 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279309192 ingress_bytes=389179392 link_bytes=389179392 token_bytes=558388104 weight_movement_bytes=110100480 total_nvlink_bytes=668488584 weight_to_token_ratio=0.197176 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768469717e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=228760 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278877592 ingress_bytes=388749312 link_bytes=388749312 token_bytes=557526424 weight_movement_bytes=110100480 total_nvlink_bytes=667626904 weight_to_token_ratio=0.197480 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760989918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=39006 dup_rows=20940 dup_rows_remote=18266 dedup_ratio=0.319519 router_tokens_per_expert=[456, 523, 519, 515, 520, 544, 513, 490, 504, 531, 513, 521, 518, 513, 549, 510, 533, 511, 489, 492, 512, 494, 482, 514, 488, 544, 518, 519, 529, 532, 522, 505, 520, 553, 529, 534, 515, 492, 527, 488, 494, 504, 487, 490, 536, 517, 498, 455, 545, 507, 483, 482, 539, 480, 536, 508, 497, 551, 492, 548, 540, 464, 525, 509] payload_rows_per_dst=[5476, 5764, 5512, 5582, 5620, 5502, 5550, 5590] dup_rows_per_dst=[2372, 2866, 2484, 2732, 2754, 2460, 2598, 2674] combine_rows_per_src=[5476, 5764, 5512, 5582, 5620, 5502, 5550, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44352 payload_rows_remote=38752 dup_rows=21184 dup_rows_remote=18602 dedup_ratio=0.323242 router_tokens_per_expert=[530, 532, 533, 517, 514, 534, 470, 476, 533, 490, 510, 481, 474, 532, 514, 522, 522, 509, 510, 486, 487, 506, 526, 545, 512, 501, 523, 540, 485, 503, 526, 511, 497, 523, 501, 490, 514, 491, 496, 524, 496, 509, 506, 513, 493, 558, 506, 478, 532, 506, 544, 501, 525, 514, 524, 517, 535, 553, 523, 516, 470, 514, 531, 514] payload_rows_per_dst=[5546, 5552, 5600, 5526, 5500, 5456, 5596, 5576] dup_rows_per_dst=[2666, 2560, 2582, 2676, 2572, 2662, 2730, 2736] combine_rows_per_src=[5546, 5552, 5600, 5526, 5500, 5456, 5596, 5576] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44354 payload_rows_remote=38764 dup_rows=21182 dup_rows_remote=18486 dedup_ratio=0.323212 router_tokens_per_expert=[512, 525, 492, 540, 520, 524, 464, 478, 519, 508, 517, 506, 536, 549, 525, 548, 502, 512, 486, 501, 496, 516, 502, 500, 483, 539, 463, 510, 538, 502, 523, 520, 503, 495, 467, 507, 532, 529, 531, 539, 503, 536, 525, 514, 538, 528, 483, 544, 485, 513, 518, 532, 509, 450, 534, 457, 520, 497, 502, 533, 498, 519, 548, 523] payload_rows_per_dst=[5590, 5646, 5518, 5402, 5540, 5686, 5654, 5318] dup_rows_per_dst=[2696, 2778, 2512, 2594, 2642, 2796, 2772, 2392] combine_rows_per_src=[5590, 5646, 5518, 5402, 5540, 5686, 5654, 5318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278405120 dispatch_meta_bytes=229208 combine_payload_bytes=278405120 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278634328 ingress_bytes=608706560 link_bytes=608706560 token_bytes=557039448 weight_movement_bytes=330301440 total_nvlink_bytes=887340888 weight_to_token_ratio=0.592959 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058640823e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44676 payload_rows_remote=39106 dup_rows=20860 dup_rows_remote=18394 dedup_ratio=0.318298 router_tokens_per_expert=[481, 529, 546, 481, 521, 527, 526, 534, 533, 506, 494, 530, 548, 538, 525, 516, 493, 485, 501, 511, 471, 510, 511, 518, 487, 540, 513, 513, 509, 499, 490, 540, 523, 508, 517, 514, 516, 542, 497, 541, 529, 516, 469, 488, 531, 491, 500, 517, 539, 512, 474, 528, 481, 516, 484, 484, 520, 500, 546, 503, 551, 523, 496, 486] payload_rows_per_dst=[5642, 5630, 5538, 5590, 5648, 5508, 5570, 5550] dup_rows_per_dst=[2648, 2750, 2462, 2592, 2668, 2574, 2466, 2700] combine_rows_per_src=[5642, 5630, 5538, 5590, 5648, 5508, 5570, 5550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277387264 dispatch_meta_bytes=229024 combine_payload_bytes=277387264 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277616288 ingress_bytes=607688704 link_bytes=607688704 token_bytes=555003552 weight_movement_bytes=330301440 total_nvlink_bytes=885304992 weight_to_token_ratio=0.595134 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056870604e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279982080 dispatch_meta_bytes=229552 combine_payload_bytes=279982080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280211632 ingress_bytes=390082560 link_bytes=390082560 token_bytes=560193712 weight_movement_bytes=110100480 total_nvlink_bytes=670294192 weight_to_token_ratio=0.196540 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784177294e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277645312 dispatch_meta_bytes=229192 combine_payload_bytes=277645312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277874504 ingress_bytes=387745792 link_bytes=387745792 token_bytes=555519816 weight_movement_bytes=110100480 total_nvlink_bytes=665620296 weight_to_token_ratio=0.198194 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.743537055e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277860352 dispatch_meta_bytes=229000 combine_payload_bytes=277860352 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278089352 ingress_bytes=498061312 link_bytes=498061312 token_bytes=555949704 weight_movement_bytes=220200960 total_nvlink_bytes=776150664 weight_to_token_ratio=0.396081 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662105386e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=229088 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279824096 ingress_bytes=719996928 link_bytes=719996928 token_bytes=559419104 weight_movement_bytes=440401920 total_nvlink_bytes=999821024 weight_to_token_ratio=0.787249 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252193077e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38842 dup_rows=21124 dup_rows_remote=18340 dedup_ratio=0.322327 router_tokens_per_expert=[526, 529, 511, 491, 520, 557, 503, 533, 516, 513, 529, 507, 518, 493, 514, 480, 496, 541, 529, 528, 475, 484, 516, 535, 533, 486, 505, 524, 528, 557, 529, 515, 517, 498, 527, 527, 515, 519, 501, 491, 490, 513, 548, 467, 515, 523, 460, 523, 508, 533, 514, 487, 474, 496, 455, 502, 524, 498, 503, 528, 509, 527, 510, 545] payload_rows_per_dst=[5656, 5508, 5554, 5570, 5556, 5514, 5452, 5602] dup_rows_per_dst=[2684, 2632, 2654, 2784, 2634, 2564, 2486, 2686] combine_rows_per_src=[5656, 5508, 5554, 5570, 5556, 5514, 5452, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277774336 dispatch_meta_bytes=229416 combine_payload_bytes=277774336 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278003752 ingress_bytes=497975296 link_bytes=497975296 token_bytes=555778088 weight_movement_bytes=220200960 total_nvlink_bytes=775979048 weight_to_token_ratio=0.396203 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.660609426e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277860352 dispatch_meta_bytes=229000 combine_payload_bytes=277860352 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278089352 ingress_bytes=498061312 link_bytes=498061312 token_bytes=555949704 weight_movement_bytes=220200960 total_nvlink_bytes=776150664 weight_to_token_ratio=0.396081 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662105386e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44262 payload_rows_remote=38734 dup_rows=21274 dup_rows_remote=18564 dedup_ratio=0.324615 router_tokens_per_expert=[496, 512, 516, 531, 476, 486, 523, 483, 485, 503, 478, 538, 505, 515, 475, 524, 515, 514, 513, 529, 573, 484, 529, 520, 510, 513, 504, 540, 511, 500, 488, 496, 521, 500, 506, 524, 539, 525, 509, 502, 500, 464, 503, 521, 546, 510, 543, 532, 502, 526, 529, 513, 501, 520, 555, 495, 520, 488, 520, 531, 523, 482, 520, 513] payload_rows_per_dst=[5476, 5512, 5518, 5508, 5566, 5528, 5618, 5536] dup_rows_per_dst=[2570, 2534, 2836, 2616, 2686, 2710, 2664, 2658] combine_rows_per_src=[5476, 5512, 5518, 5508, 5566, 5528, 5618, 5536] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38952 dup_rows=21070 dup_rows_remote=18406 dedup_ratio=0.321503 router_tokens_per_expert=[511, 516, 510, 536, 530, 535, 497, 534, 504, 492, 514, 481, 472, 496, 523, 509, 521, 504, 511, 511, 532, 488, 502, 520, 539, 531, 483, 492, 513, 493, 534, 500, 463, 486, 514, 503, 533, 527, 505, 485, 521, 472, 494, 496, 515, 544, 526, 498, 489, 517, 479, 516, 530, 536, 521, 536, 534, 560, 493, 529, 548, 499, 518, 547] payload_rows_per_dst=[5718, 5424, 5514, 5524, 5484, 5530, 5578, 5694] dup_rows_per_dst=[2620, 2558, 2664, 2646, 2548, 2602, 2670, 2762] combine_rows_per_src=[5718, 5424, 5514, 5524, 5484, 5530, 5578, 5694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38884 dup_rows=21084 dup_rows_remote=18448 dedup_ratio=0.321716 router_tokens_per_expert=[522, 502, 507, 476, 515, 487, 515, 550, 514, 511, 499, 530, 494, 536, 502, 516, 487, 491, 534, 526, 455, 527, 515, 511, 472, 503, 532, 508, 522, 548, 521, 506, 510, 531, 569, 517, 518, 500, 482, 524, 467, 553, 520, 512, 479, 557, 540, 536, 493, 481, 535, 472, 495, 468, 515, 510, 491, 502, 492, 526, 536, 529, 539, 535] payload_rows_per_dst=[5576, 5568, 5488, 5528, 5630, 5622, 5500, 5540] dup_rows_per_dst=[2572, 2636, 2604, 2696, 2672, 2706, 2438, 2760] combine_rows_per_src=[5576, 5568, 5488, 5528, 5630, 5622, 5500, 5540] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38840 dup_rows=21158 dup_rows_remote=18462 dedup_ratio=0.322845 router_tokens_per_expert=[509, 533, 488, 493, 528, 492, 510, 524, 544, 529, 515, 513, 501, 520, 487, 546, 547, 483, 537, 494, 502, 529, 510, 457, 520, 510, 467, 493, 537, 515, 489, 558, 497, 516, 513, 522, 542, 531, 505, 491, 516, 499, 528, 471, 515, 507, 488, 514, 503, 516, 548, 549, 516, 500, 497, 531, 544, 498, 519, 509, 490, 514, 488, 511] payload_rows_per_dst=[5486, 5558, 5552, 5588, 5538, 5486, 5626, 5544] dup_rows_per_dst=[2668, 2752, 2566, 2590, 2696, 2590, 2694, 2602] combine_rows_per_src=[5486, 5558, 5552, 5588, 5538, 5486, 5626, 5544] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280311808 dispatch_meta_bytes=230000 combine_payload_bytes=280311808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280541808 ingress_bytes=390412288 link_bytes=390412288 token_bytes=560853616 weight_movement_bytes=110100480 total_nvlink_bytes=670954096 weight_to_token_ratio=0.196309 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789911806e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=228728 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278648184 ingress_bytes=498620416 link_bytes=498620416 token_bytes=557067640 weight_movement_bytes=220200960 total_nvlink_bytes=777268600 weight_to_token_ratio=0.395286 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671829124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229432 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279437368 ingress_bytes=389308416 link_bytes=389308416 token_bytes=558645304 weight_movement_bytes=110100480 total_nvlink_bytes=668745784 weight_to_token_ratio=0.197085 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770713657e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38876 dup_rows=21072 dup_rows_remote=18246 dedup_ratio=0.321533 router_tokens_per_expert=[483, 524, 518, 487, 508, 506, 495, 507, 509, 508, 525, 540, 482, 556, 527, 498, 498, 492, 492, 503, 516, 505, 503, 455, 527, 529, 503, 527, 510, 542, 526, 527, 498, 516, 540, 517, 521, 489, 517, 507, 509, 491, 529, 503, 503, 505, 498, 522, 522, 489, 518, 505, 496, 507, 529, 522, 482, 533, 524, 528, 516, 548, 537, 519] payload_rows_per_dst=[5572, 5356, 5656, 5578, 5570, 5550, 5594, 5588] dup_rows_per_dst=[2484, 2380, 2826, 2722, 2558, 2570, 2706, 2826] combine_rows_per_src=[5572, 5356, 5656, 5578, 5570, 5550, 5594, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44828 payload_rows_remote=39192 dup_rows=20708 dup_rows_remote=18022 dedup_ratio=0.315979 router_tokens_per_expert=[516, 538, 503, 560, 510, 532, 496, 536, 545, 505, 500, 523, 542, 519, 505, 486, 517, 495, 515, 483, 466, 510, 506, 512, 500, 488, 519, 536, 536, 540, 494, 484, 511, 490, 551, 523, 487, 519, 542, 482, 516, 502, 536, 519, 503, 534, 489, 562, 547, 474, 521, 489, 507, 498, 517, 499, 493, 480, 521, 485, 490, 516, 525, 523] payload_rows_per_dst=[5610, 5584, 5610, 5660, 5672, 5636, 5530, 5526] dup_rows_per_dst=[2772, 2666, 2398, 2534, 2538, 2686, 2574, 2540] combine_rows_per_src=[5610, 5584, 5610, 5660, 5672, 5636, 5530, 5526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44538 payload_rows_remote=38976 dup_rows=20998 dup_rows_remote=18316 dedup_ratio=0.320404 router_tokens_per_expert=[493, 499, 504, 508, 526, 508, 515, 492, 516, 509, 512, 547, 517, 519, 505, 497, 520, 523, 512, 527, 527, 513, 524, 514, 573, 548, 532, 534, 529, 513, 497, 510, 496, 515, 496, 504, 539, 486, 486, 501, 481, 531, 543, 498, 519, 483, 468, 519, 507, 492, 515, 492, 516, 497, 518, 544, 511, 517, 467, 557, 538, 501, 478, 490] payload_rows_per_dst=[5556, 5562, 5618, 5696, 5484, 5614, 5520, 5488] dup_rows_per_dst=[2534, 2682, 2702, 2776, 2562, 2470, 2642, 2630] combine_rows_per_src=[5556, 5562, 5618, 5696, 5484, 5614, 5520, 5488] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38852 dup_rows=21144 dup_rows_remote=18542 dedup_ratio=0.322632 router_tokens_per_expert=[473, 501, 529, 499, 492, 510, 500, 517, 519, 481, 502, 558, 547, 543, 510, 506, 511, 485, 513, 536, 507, 541, 514, 512, 503, 527, 506, 542, 489, 503, 523, 509, 516, 487, 471, 494, 523, 530, 532, 518, 515, 559, 510, 498, 494, 539, 493, 490, 531, 534, 503, 537, 514, 487, 532, 538, 494, 509, 507, 495, 485, 538, 509, 478] payload_rows_per_dst=[5436, 5626, 5546, 5596, 5540, 5552, 5594, 5502] dup_rows_per_dst=[2606, 2706, 2692, 2608, 2602, 2644, 2758, 2528] combine_rows_per_src=[5436, 5626, 5546, 5596, 5540, 5552, 5594, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277645312 dispatch_meta_bytes=229192 combine_payload_bytes=277645312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277874504 ingress_bytes=387745792 link_bytes=387745792 token_bytes=555519816 weight_movement_bytes=110100480 total_nvlink_bytes=665620296 weight_to_token_ratio=0.198194 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.743537055e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229328 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278949840 ingress_bytes=388820992 link_bytes=388820992 token_bytes=557670352 weight_movement_bytes=110100480 total_nvlink_bytes=667770832 weight_to_token_ratio=0.197429 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762236552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278405120 dispatch_meta_bytes=229208 combine_payload_bytes=278405120 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278634328 ingress_bytes=608706560 link_bytes=608706560 token_bytes=557039448 weight_movement_bytes=330301440 total_nvlink_bytes=887340888 weight_to_token_ratio=0.592959 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058640823e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44212 payload_rows_remote=38698 dup_rows=21324 dup_rows_remote=18558 dedup_ratio=0.325378 router_tokens_per_expert=[511, 525, 500, 568, 464, 507, 530, 538, 517, 522, 505, 513, 474, 504, 499, 519, 485, 505, 545, 536, 502, 478, 494, 502, 502, 514, 478, 510, 490, 560, 525, 540, 524, 533, 500, 524, 537, 477, 523, 534, 535, 512, 527, 513, 487, 493, 504, 505, 485, 506, 516, 511, 517, 472, 520, 528, 509, 465, 515, 529, 520, 524, 528, 533] payload_rows_per_dst=[5342, 5586, 5630, 5552, 5572, 5518, 5498, 5514] dup_rows_per_dst=[2542, 2520, 2866, 2686, 2698, 2634, 2612, 2766] combine_rows_per_src=[5342, 5586, 5630, 5552, 5572, 5518, 5498, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44550 payload_rows_remote=38896 dup_rows=20986 dup_rows_remote=18304 dedup_ratio=0.320221 router_tokens_per_expert=[513, 532, 521, 527, 555, 513, 502, 503, 522, 490, 521, 497, 508, 517, 494, 519, 537, 533, 541, 511, 495, 513, 534, 504, 493, 504, 518, 515, 528, 475, 523, 533, 509, 422, 500, 502, 511, 491, 555, 516, 497, 562, 489, 529, 534, 499, 531, 530, 491, 469, 522, 545, 535, 499, 499, 492, 508, 497, 522, 482, 493, 526, 513, 507] payload_rows_per_dst=[5598, 5534, 5654, 5572, 5504, 5652, 5488, 5548] dup_rows_per_dst=[2734, 2602, 2682, 2606, 2508, 2690, 2616, 2548] combine_rows_per_src=[5598, 5534, 5654, 5572, 5504, 5652, 5488, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38958 dup_rows=20992 dup_rows_remote=18138 dedup_ratio=0.320312 router_tokens_per_expert=[521, 527, 485, 539, 530, 511, 525, 513, 501, 535, 450, 545, 537, 503, 523, 511, 471, 526, 515, 488, 497, 510, 499, 507, 496, 520, 513, 507, 513, 506, 504, 525, 521, 537, 517, 505, 463, 523, 541, 522, 491, 530, 516, 539, 495, 524, 501, 544, 507, 474, 502, 583, 510, 514, 478, 481, 493, 500, 526, 492, 534, 542, 487, 523] payload_rows_per_dst=[5586, 5662, 5612, 5618, 5526, 5524, 5470, 5546] dup_rows_per_dst=[2854, 2770, 2414, 2558, 2594, 2526, 2628, 2648] combine_rows_per_src=[5586, 5662, 5612, 5618, 5526, 5524, 5470, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277387264 dispatch_meta_bytes=229024 combine_payload_bytes=277387264 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277616288 ingress_bytes=607688704 link_bytes=607688704 token_bytes=555003552 weight_movement_bytes=330301440 total_nvlink_bytes=885304992 weight_to_token_ratio=0.595134 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056870604e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278663168 dispatch_meta_bytes=228488 combine_payload_bytes=278663168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278891656 ingress_bytes=388763648 link_bytes=388763648 token_bytes=557554824 weight_movement_bytes=110100480 total_nvlink_bytes=667655304 weight_to_token_ratio=0.197470 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761239245e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=38940 dup_rows=21008 dup_rows_remote=18374 dedup_ratio=0.320557 router_tokens_per_expert=[498, 542, 496, 519, 514, 509, 486, 499, 518, 484, 504, 510, 483, 534, 488, 494, 497, 532, 528, 543, 528, 529, 482, 471, 515, 517, 505, 477, 492, 511, 486, 546, 516, 527, 518, 511, 541, 519, 524, 530, 505, 493, 537, 519, 495, 502, 499, 532, 513, 539, 534, 514, 473, 499, 515, 524, 539, 476, 541, 507, 534, 549, 511, 495] payload_rows_per_dst=[5598, 5538, 5530, 5514, 5642, 5556, 5588, 5562] dup_rows_per_dst=[2528, 2492, 2690, 2584, 2730, 2608, 2634, 2742] combine_rows_per_src=[5598, 5538, 5530, 5514, 5642, 5556, 5588, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280928256 dispatch_meta_bytes=228856 combine_payload_bytes=280928256 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=281157112 ingress_bytes=501129216 link_bytes=501129216 token_bytes=562085368 weight_movement_bytes=220200960 total_nvlink_bytes=782286328 weight_to_token_ratio=0.391757 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.715461282e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=39034 dup_rows=20946 dup_rows_remote=18374 dedup_ratio=0.319611 router_tokens_per_expert=[515, 556, 526, 512, 528, 511, 477, 487, 544, 511, 529, 555, 527, 544, 491, 523, 486, 489, 550, 514, 510, 500, 491, 493, 503, 536, 509, 516, 481, 523, 495, 501, 502, 559, 509, 516, 502, 553, 488, 474, 515, 525, 500, 508, 531, 488, 518, 468, 496, 493, 504, 490, 533, 486, 525, 530, 513, 564, 520, 499, 524, 480, 510, 512] payload_rows_per_dst=[5522, 5668, 5506, 5556, 5612, 5614, 5556, 5556] dup_rows_per_dst=[2702, 2780, 2560, 2572, 2594, 2492, 2558, 2688] combine_rows_per_src=[5522, 5668, 5506, 5556, 5612, 5614, 5556, 5556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=229168 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279609136 ingress_bytes=499580928 link_bytes=499580928 token_bytes=558989104 weight_movement_bytes=220200960 total_nvlink_bytes=779190064 weight_to_token_ratio=0.393927 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688534007e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=229576 combine_payload_bytes=278491136 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278720712 ingress_bytes=608792576 link_bytes=608792576 token_bytes=557211848 weight_movement_bytes=330301440 total_nvlink_bytes=887513288 weight_to_token_ratio=0.592775 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058790419e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=228800 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279035328 ingress_bytes=499007488 link_bytes=499007488 token_bytes=557841856 weight_movement_bytes=220200960 total_nvlink_bytes=778042816 weight_to_token_ratio=0.394737 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678560943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44828 payload_rows_remote=39192 dup_rows=20708 dup_rows_remote=18022 dedup_ratio=0.315979 router_tokens_per_expert=[516, 538, 503, 560, 510, 532, 496, 536, 545, 505, 500, 523, 542, 519, 505, 486, 517, 495, 515, 483, 466, 510, 506, 512, 500, 488, 519, 536, 536, 540, 494, 484, 511, 490, 551, 523, 487, 519, 542, 482, 516, 502, 536, 519, 503, 534, 489, 562, 547, 474, 521, 489, 507, 498, 517, 499, 493, 480, 521, 485, 490, 516, 525, 523] payload_rows_per_dst=[5610, 5584, 5610, 5660, 5672, 5636, 5530, 5526] dup_rows_per_dst=[2772, 2666, 2398, 2534, 2538, 2686, 2574, 2540] combine_rows_per_src=[5610, 5584, 5610, 5660, 5672, 5636, 5530, 5526] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=228384 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279479328 ingress_bytes=389351424 link_bytes=389351424 token_bytes=558730272 weight_movement_bytes=110100480 total_nvlink_bytes=668830752 weight_to_token_ratio=0.197055 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771461636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44642 payload_rows_remote=39060 dup_rows=20894 dup_rows_remote=18328 dedup_ratio=0.318817 router_tokens_per_expert=[514, 522, 529, 501, 494, 541, 488, 510, 534, 507, 494, 521, 501, 514, 493, 510, 483, 543, 530, 489, 517, 509, 529, 476, 517, 536, 504, 497, 517, 502, 560, 514, 538, 483, 488, 481, 477, 515, 549, 499, 522, 545, 528, 513, 516, 510, 508, 506, 449, 527, 503, 524, 513, 508, 507, 557, 498, 531, 519, 530, 508, 525, 484, 511] payload_rows_per_dst=[5622, 5582, 5612, 5610, 5570, 5608, 5540, 5498] dup_rows_per_dst=[2576, 2566, 2540, 2684, 2490, 2688, 2636, 2714] combine_rows_per_src=[5622, 5582, 5612, 5610, 5570, 5608, 5540, 5498] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38892 dup_rows=21118 dup_rows_remote=18468 dedup_ratio=0.322235 router_tokens_per_expert=[503, 505, 540, 475, 542, 531, 494, 498, 493, 509, 517, 518, 500, 502, 479, 514, 500, 548, 531, 500, 473, 470, 532, 534, 479, 515, 492, 521, 456, 494, 529, 496, 557, 513, 497, 562, 541, 481, 468, 537, 555, 533, 567, 552, 517, 504, 497, 522, 545, 501, 500, 500, 504, 500, 495, 547, 530, 515, 484, 512, 520, 500, 478, 544] payload_rows_per_dst=[5522, 5424, 5526, 5474, 5654, 5654, 5584, 5580] dup_rows_per_dst=[2654, 2640, 2650, 2490, 2658, 2840, 2600, 2586] combine_rows_per_src=[5522, 5424, 5526, 5474, 5654, 5654, 5584, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38852 dup_rows=21144 dup_rows_remote=18542 dedup_ratio=0.322632 router_tokens_per_expert=[473, 501, 529, 499, 492, 510, 500, 517, 519, 481, 502, 558, 547, 543, 510, 506, 511, 485, 513, 536, 507, 541, 514, 512, 503, 527, 506, 542, 489, 503, 523, 509, 516, 487, 471, 494, 523, 530, 532, 518, 515, 559, 510, 498, 494, 539, 493, 490, 531, 534, 503, 537, 514, 487, 532, 538, 494, 509, 507, 495, 485, 538, 509, 478] payload_rows_per_dst=[5436, 5626, 5546, 5596, 5540, 5552, 5594, 5502] dup_rows_per_dst=[2606, 2706, 2692, 2608, 2602, 2644, 2758, 2528] combine_rows_per_src=[5436, 5626, 5546, 5596, 5540, 5552, 5594, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=229256 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279351176 ingress_bytes=389222400 link_bytes=389222400 token_bytes=558473096 weight_movement_bytes=110100480 total_nvlink_bytes=668573576 weight_to_token_ratio=0.197146 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769217697e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229632 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280025344 ingress_bytes=610097152 link_bytes=610097152 token_bytes=559821056 weight_movement_bytes=330301440 total_nvlink_bytes=890122496 weight_to_token_ratio=0.590013 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061059291e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=229440 combine_payload_bytes=278777856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279007296 ingress_bytes=388878336 link_bytes=388878336 token_bytes=557785152 weight_movement_bytes=110100480 total_nvlink_bytes=667885632 weight_to_token_ratio=0.197389 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763233858e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38916 dup_rows=21070 dup_rows_remote=18396 dedup_ratio=0.321503 router_tokens_per_expert=[566, 488, 504, 490, 537, 502, 499, 511, 515, 483, 535, 524, 498, 561, 467, 486, 500, 481, 542, 529, 493, 462, 515, 492, 506, 517, 544, 532, 498, 482, 493, 501, 558, 566, 528, 517, 505, 514, 523, 515, 515, 531, 490, 554, 533, 502, 506, 511, 551, 497, 504, 505, 475, 540, 504, 514, 524, 519, 539, 522, 507, 483, 486, 477] payload_rows_per_dst=[5566, 5548, 5458, 5564, 5710, 5542, 5528, 5550] dup_rows_per_dst=[2628, 2590, 2570, 2582, 2742, 2632, 2652, 2674] combine_rows_per_src=[5566, 5548, 5458, 5564, 5710, 5542, 5528, 5550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38924 dup_rows=21042 dup_rows_remote=18478 dedup_ratio=0.321075 router_tokens_per_expert=[532, 503, 540, 532, 444, 520, 510, 487, 513, 494, 499, 495, 491, 480, 513, 515, 521, 525, 448, 533, 499, 515, 519, 479, 531, 549, 509, 489, 520, 526, 497, 532, 527, 562, 497, 532, 500, 511, 546, 511, 509, 502, 528, 541, 485, 501, 499, 502, 534, 508, 508, 520, 543, 439, 530, 498, 535, 536, 520, 540, 501, 487, 529, 527] payload_rows_per_dst=[5548, 5446, 5532, 5672, 5620, 5570, 5530, 5576] dup_rows_per_dst=[2588, 2554, 2546, 2634, 2752, 2564, 2630, 2774] combine_rows_per_src=[5548, 5446, 5532, 5672, 5620, 5570, 5530, 5576] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38994 dup_rows=21054 dup_rows_remote=18384 dedup_ratio=0.321259 router_tokens_per_expert=[536, 492, 522, 522, 489, 523, 509, 438, 526, 481, 506, 531, 492, 536, 533, 543, 495, 502, 479, 521, 529, 501, 498, 474, 522, 528, 439, 524, 478, 489, 549, 489, 520, 517, 522, 499, 499, 502, 531, 489, 531, 545, 525, 540, 546, 538, 462, 497, 536, 527, 522, 517, 507, 503, 494, 564, 498, 527, 539, 506, 530, 488, 526, 525] payload_rows_per_dst=[5510, 5612, 5544, 5466, 5488, 5716, 5610, 5536] dup_rows_per_dst=[2552, 2684, 2454, 2570, 2670, 2652, 2730, 2742] combine_rows_per_src=[5510, 5612, 5544, 5466, 5488, 5716, 5610, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44408 payload_rows_remote=38926 dup_rows=21128 dup_rows_remote=18442 dedup_ratio=0.322388 router_tokens_per_expert=[477, 523, 487, 482, 549, 513, 508, 509, 506, 489, 513, 518, 475, 521, 552, 510, 493, 548, 520, 479, 568, 520, 505, 517, 543, 518, 479, 487, 480, 494, 502, 536, 496, 499, 496, 513, 556, 515, 553, 524, 481, 504, 520, 547, 516, 519, 502, 519, 543, 553, 504, 504, 517, 485, 534, 486, 495, 548, 530, 512, 474, 500, 502, 500] payload_rows_per_dst=[5480, 5482, 5628, 5544, 5602, 5620, 5566, 5486] dup_rows_per_dst=[2616, 2686, 2672, 2534, 2702, 2596, 2686, 2636] combine_rows_per_src=[5480, 5482, 5628, 5544, 5602, 5620, 5566, 5486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280928256 dispatch_meta_bytes=228856 combine_payload_bytes=280928256 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=281157112 ingress_bytes=501129216 link_bytes=501129216 token_bytes=562085368 weight_movement_bytes=220200960 total_nvlink_bytes=782286328 weight_to_token_ratio=0.391757 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.715461282e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38876 dup_rows=21072 dup_rows_remote=18246 dedup_ratio=0.321533 router_tokens_per_expert=[483, 524, 518, 487, 508, 506, 495, 507, 509, 508, 525, 540, 482, 556, 527, 498, 498, 492, 492, 503, 516, 505, 503, 455, 527, 529, 503, 527, 510, 542, 526, 527, 498, 516, 540, 517, 521, 489, 517, 507, 509, 491, 529, 503, 503, 505, 498, 522, 522, 489, 518, 505, 496, 507, 529, 522, 482, 533, 524, 528, 516, 548, 537, 519] payload_rows_per_dst=[5572, 5356, 5656, 5578, 5570, 5550, 5594, 5588] dup_rows_per_dst=[2484, 2380, 2826, 2722, 2558, 2570, 2706, 2826] combine_rows_per_src=[5572, 5356, 5656, 5578, 5570, 5550, 5594, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279982080 dispatch_meta_bytes=229552 combine_payload_bytes=279982080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280211632 ingress_bytes=390082560 link_bytes=390082560 token_bytes=560193712 weight_movement_bytes=110100480 total_nvlink_bytes=670294192 weight_to_token_ratio=0.196540 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784177294e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=229576 combine_payload_bytes=278491136 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278720712 ingress_bytes=608792576 link_bytes=608792576 token_bytes=557211848 weight_movement_bytes=330301440 total_nvlink_bytes=887513288 weight_to_token_ratio=0.592775 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058790419e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38952 dup_rows=21070 dup_rows_remote=18406 dedup_ratio=0.321503 router_tokens_per_expert=[511, 516, 510, 536, 530, 535, 497, 534, 504, 492, 514, 481, 472, 496, 523, 509, 521, 504, 511, 511, 532, 488, 502, 520, 539, 531, 483, 492, 513, 493, 534, 500, 463, 486, 514, 503, 533, 527, 505, 485, 521, 472, 494, 496, 515, 544, 526, 498, 489, 517, 479, 516, 530, 536, 521, 536, 534, 560, 493, 529, 548, 499, 518, 547] payload_rows_per_dst=[5718, 5424, 5514, 5524, 5484, 5530, 5578, 5694] dup_rows_per_dst=[2620, 2558, 2664, 2646, 2548, 2602, 2670, 2762] combine_rows_per_src=[5718, 5424, 5514, 5524, 5484, 5530, 5578, 5694] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38652 dup_rows=21122 dup_rows_remote=18126 dedup_ratio=0.322296 router_tokens_per_expert=[523, 500, 505, 498, 499, 515, 518, 510, 502, 480, 521, 483, 511, 467, 528, 500, 492, 523, 513, 504, 526, 506, 530, 491, 518, 509, 540, 504, 534, 517, 511, 523, 528, 502, 524, 519, 522, 505, 502, 498, 566, 530, 510, 512, 491, 509, 487, 530, 561, 518, 484, 509, 506, 531, 529, 547, 495, 508, 522, 496, 522, 494, 512, 498] payload_rows_per_dst=[5762, 5710, 5766, 5462, 5266, 5634, 5608, 5206] dup_rows_per_dst=[2996, 2772, 2936, 2570, 2130, 2658, 2704, 2356] combine_rows_per_src=[5762, 5710, 5766, 5462, 5266, 5634, 5608, 5206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=229248 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279179136 ingress_bytes=499150848 link_bytes=499150848 token_bytes=558129024 weight_movement_bytes=220200960 total_nvlink_bytes=778329984 weight_to_token_ratio=0.394534 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681054209e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278663168 dispatch_meta_bytes=228488 combine_payload_bytes=278663168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278891656 ingress_bytes=388763648 link_bytes=388763648 token_bytes=557554824 weight_movement_bytes=110100480 total_nvlink_bytes=667655304 weight_to_token_ratio=0.197470 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761239245e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38870 dup_rows=21108 dup_rows_remote=18580 dedup_ratio=0.322083 router_tokens_per_expert=[520, 529, 543, 492, 520, 484, 511, 505, 485, 544, 517, 507, 512, 512, 517, 518, 498, 497, 492, 479, 542, 502, 530, 518, 562, 530, 554, 457, 518, 544, 519, 538, 477, 535, 497, 537, 525, 517, 511, 474, 516, 495, 487, 500, 502, 510, 488, 528, 512, 544, 494, 475, 511, 535, 473, 499, 488, 520, 513, 509, 553, 510, 536, 501] payload_rows_per_dst=[5588, 5554, 5510, 5648, 5554, 5478, 5558, 5538] dup_rows_per_dst=[2620, 2670, 2606, 2796, 2592, 2574, 2528, 2722] combine_rows_per_src=[5588, 5554, 5510, 5648, 5554, 5478, 5558, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44458 payload_rows_remote=38918 dup_rows=21078 dup_rows_remote=18398 dedup_ratio=0.321625 router_tokens_per_expert=[514, 509, 481, 502, 485, 486, 501, 528, 492, 525, 490, 516, 500, 515, 543, 500, 505, 484, 468, 516, 481, 519, 504, 519, 550, 559, 505, 472, 520, 504, 498, 502, 529, 531, 529, 540, 519, 488, 500, 566, 517, 500, 515, 534, 526, 541, 501, 513, 515, 522, 539, 521, 468, 467, 464, 521, 527, 503, 528, 553, 525, 535, 504, 534] payload_rows_per_dst=[5448, 5564, 5516, 5540, 5652, 5634, 5528, 5576] dup_rows_per_dst=[2564, 2598, 2476, 2680, 2752, 2660, 2506, 2842] combine_rows_per_src=[5448, 5564, 5516, 5540, 5652, 5634, 5528, 5576] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229608 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279236840 ingress_bytes=499208192 link_bytes=499208192 token_bytes=558244072 weight_movement_bytes=220200960 total_nvlink_bytes=778445032 weight_to_token_ratio=0.394453 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682051515e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=229512 combine_payload_bytes=279508992 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279738504 ingress_bytes=609810432 link_bytes=609810432 token_bytes=559247496 weight_movement_bytes=330301440 total_nvlink_bytes=889548936 weight_to_token_ratio=0.590618 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060560638e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279021568 dispatch_meta_bytes=229472 combine_payload_bytes=279021568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279251040 ingress_bytes=389122048 link_bytes=389122048 token_bytes=558272608 weight_movement_bytes=110100480 total_nvlink_bytes=668373088 weight_to_token_ratio=0.197216 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767472411e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277057536 dispatch_meta_bytes=227112 combine_payload_bytes=277057536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277284648 ingress_bytes=387158016 link_bytes=387158016 token_bytes=554342184 weight_movement_bytes=110100480 total_nvlink_bytes=664442664 weight_to_token_ratio=0.198615 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.733314664e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38924 dup_rows=21042 dup_rows_remote=18478 dedup_ratio=0.321075 router_tokens_per_expert=[532, 503, 540, 532, 444, 520, 510, 487, 513, 494, 499, 495, 491, 480, 513, 515, 521, 525, 448, 533, 499, 515, 519, 479, 531, 549, 509, 489, 520, 526, 497, 532, 527, 562, 497, 532, 500, 511, 546, 511, 509, 502, 528, 541, 485, 501, 499, 502, 534, 508, 508, 520, 543, 439, 530, 498, 535, 536, 520, 540, 501, 487, 529, 527] payload_rows_per_dst=[5548, 5446, 5532, 5672, 5620, 5570, 5530, 5576] dup_rows_per_dst=[2588, 2554, 2546, 2634, 2752, 2564, 2630, 2774] combine_rows_per_src=[5548, 5446, 5532, 5672, 5620, 5570, 5530, 5576] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44658 payload_rows_remote=39080 dup_rows=20878 dup_rows_remote=18134 dedup_ratio=0.318573 router_tokens_per_expert=[486, 514, 460, 518, 488, 532, 527, 510, 509, 488, 538, 501, 463, 486, 544, 504, 520, 492, 561, 528, 510, 531, 519, 500, 510, 529, 535, 508, 535, 511, 523, 486, 546, 484, 529, 530, 495, 504, 541, 523, 491, 501, 507, 498, 517, 489, 482, 518, 533, 504, 502, 512, 499, 522, 484, 530, 535, 531, 492, 531, 521, 522, 525, 504] payload_rows_per_dst=[5508, 5590, 5578, 5688, 5628, 5478, 5568, 5620] dup_rows_per_dst=[2562, 2476, 2744, 2586, 2676, 2528, 2604, 2702] combine_rows_per_src=[5508, 5590, 5578, 5688, 5628, 5478, 5568, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229432 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279437368 ingress_bytes=389308416 link_bytes=389308416 token_bytes=558645304 weight_movement_bytes=110100480 total_nvlink_bytes=668745784 weight_to_token_ratio=0.197085 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770713657e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277057536 dispatch_meta_bytes=227112 combine_payload_bytes=277057536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277284648 ingress_bytes=387158016 link_bytes=387158016 token_bytes=554342184 weight_movement_bytes=110100480 total_nvlink_bytes=664442664 weight_to_token_ratio=0.198615 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.733314664e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44538 payload_rows_remote=38976 dup_rows=20998 dup_rows_remote=18316 dedup_ratio=0.320404 router_tokens_per_expert=[493, 499, 504, 508, 526, 508, 515, 492, 516, 509, 512, 547, 517, 519, 505, 497, 520, 523, 512, 527, 527, 513, 524, 514, 573, 548, 532, 534, 529, 513, 497, 510, 496, 515, 496, 504, 539, 486, 486, 501, 481, 531, 543, 498, 519, 483, 468, 519, 507, 492, 515, 492, 516, 497, 518, 544, 511, 517, 467, 557, 538, 501, 478, 490] payload_rows_per_dst=[5556, 5562, 5618, 5696, 5484, 5614, 5520, 5488] dup_rows_per_dst=[2534, 2682, 2702, 2776, 2562, 2470, 2642, 2630] combine_rows_per_src=[5556, 5562, 5618, 5696, 5484, 5614, 5520, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38994 dup_rows=21054 dup_rows_remote=18384 dedup_ratio=0.321259 router_tokens_per_expert=[536, 492, 522, 522, 489, 523, 509, 438, 526, 481, 506, 531, 492, 536, 533, 543, 495, 502, 479, 521, 529, 501, 498, 474, 522, 528, 439, 524, 478, 489, 549, 489, 520, 517, 522, 499, 499, 502, 531, 489, 531, 545, 525, 540, 546, 538, 462, 497, 536, 527, 522, 517, 507, 503, 494, 564, 498, 527, 539, 506, 530, 488, 526, 525] payload_rows_per_dst=[5510, 5612, 5544, 5466, 5488, 5716, 5610, 5536] dup_rows_per_dst=[2552, 2684, 2454, 2570, 2670, 2652, 2730, 2742] combine_rows_per_src=[5510, 5612, 5544, 5466, 5488, 5716, 5610, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278620160 dispatch_meta_bytes=229800 combine_payload_bytes=278620160 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278849960 ingress_bytes=498821120 link_bytes=498821120 token_bytes=557470120 weight_movement_bytes=220200960 total_nvlink_bytes=777671080 weight_to_token_ratio=0.395000 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.675319697e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=229264 combine_payload_bytes=278964224 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279193488 ingress_bytes=499165184 link_bytes=499165184 token_bytes=558157712 weight_movement_bytes=220200960 total_nvlink_bytes=778358672 weight_to_token_ratio=0.394514 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681303535e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280125440 dispatch_meta_bytes=228856 combine_payload_bytes=280125440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280354296 ingress_bytes=390225920 link_bytes=390225920 token_bytes=560479736 weight_movement_bytes=110100480 total_nvlink_bytes=670580216 weight_to_token_ratio=0.196440 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786670560e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44388 payload_rows_remote=38700 dup_rows=21148 dup_rows_remote=18496 dedup_ratio=0.322693 router_tokens_per_expert=[536, 540, 469, 476, 509, 556, 540, 464, 525, 481, 531, 517, 489, 522, 509, 534, 511, 478, 484, 500, 481, 533, 498, 502, 494, 512, 518, 499, 469, 525, 576, 536, 511, 490, 479, 502, 514, 519, 502, 482, 518, 497, 503, 514, 528, 552, 516, 524, 536, 509, 527, 515, 477, 530, 545, 496, 508, 555, 520, 532, 505, 497, 532, 519] payload_rows_per_dst=[5562, 5484, 5436, 5594, 5398, 5574, 5652, 5688] dup_rows_per_dst=[2618, 2732, 2538, 2664, 2524, 2730, 2690, 2652] combine_rows_per_src=[5562, 5484, 5436, 5594, 5398, 5574, 5652, 5688] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38880 dup_rows=21026 dup_rows_remote=18308 dedup_ratio=0.320831 router_tokens_per_expert=[532, 503, 471, 515, 519, 490, 476, 499, 531, 495, 492, 526, 506, 515, 503, 524, 517, 525, 485, 540, 520, 507, 480, 523, 509, 513, 522, 534, 477, 542, 476, 538, 520, 503, 481, 513, 534, 522, 526, 575, 517, 513, 534, 515, 541, 513, 501, 511, 526, 490, 500, 526, 498, 519, 519, 522, 508, 507, 489, 480, 511, 526, 509, 514] payload_rows_per_dst=[5422, 5576, 5536, 5656, 5630, 5660, 5552, 5478] dup_rows_per_dst=[2588, 2608, 2658, 2566, 2718, 2630, 2648, 2610] combine_rows_per_src=[5422, 5576, 5536, 5656, 5630, 5660, 5552, 5478] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44678 payload_rows_remote=39196 dup_rows=20858 dup_rows_remote=18292 dedup_ratio=0.318268 router_tokens_per_expert=[550, 519, 515, 536, 530, 520, 513, 528, 495, 524, 520, 494, 515, 510, 474, 492, 503, 493, 535, 526, 533, 510, 536, 489, 495, 531, 546, 531, 494, 519, 461, 504, 536, 503, 514, 497, 483, 484, 520, 463, 487, 515, 521, 505, 532, 510, 488, 506, 515, 491, 520, 515, 536, 492, 500, 503, 560, 511, 537, 524, 511, 479, 547, 522] payload_rows_per_dst=[5744, 5482, 5632, 5542, 5526, 5532, 5542, 5678] dup_rows_per_dst=[2678, 2566, 2618, 2620, 2474, 2596, 2602, 2704] combine_rows_per_src=[5744, 5482, 5632, 5542, 5526, 5532, 5542, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229608 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279236840 ingress_bytes=499208192 link_bytes=499208192 token_bytes=558244072 weight_movement_bytes=220200960 total_nvlink_bytes=778445032 weight_to_token_ratio=0.394453 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682051515e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38916 dup_rows=21070 dup_rows_remote=18396 dedup_ratio=0.321503 router_tokens_per_expert=[566, 488, 504, 490, 537, 502, 499, 511, 515, 483, 535, 524, 498, 561, 467, 486, 500, 481, 542, 529, 493, 462, 515, 492, 506, 517, 544, 532, 498, 482, 493, 501, 558, 566, 528, 517, 505, 514, 523, 515, 515, 531, 490, 554, 533, 502, 506, 511, 551, 497, 504, 505, 475, 540, 504, 514, 524, 519, 539, 522, 507, 483, 486, 477] payload_rows_per_dst=[5566, 5548, 5458, 5564, 5710, 5542, 5528, 5550] dup_rows_per_dst=[2628, 2590, 2570, 2582, 2742, 2632, 2652, 2674] combine_rows_per_src=[5566, 5548, 5458, 5564, 5710, 5542, 5528, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=229168 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279609136 ingress_bytes=499580928 link_bytes=499580928 token_bytes=558989104 weight_movement_bytes=220200960 total_nvlink_bytes=779190064 weight_to_token_ratio=0.393927 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688534007e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=229512 combine_payload_bytes=279508992 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279738504 ingress_bytes=609810432 link_bytes=609810432 token_bytes=559247496 weight_movement_bytes=330301440 total_nvlink_bytes=889548936 weight_to_token_ratio=0.590618 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060560638e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44670 payload_rows_remote=39046 dup_rows=20866 dup_rows_remote=18106 dedup_ratio=0.318390 router_tokens_per_expert=[472, 505, 498, 490, 508, 506, 553, 507, 465, 488, 527, 508, 518, 488, 499, 508, 494, 508, 524, 524, 518, 544, 510, 476, 533, 488, 491, 512, 530, 519, 516, 505, 535, 512, 517, 539, 530, 514, 471, 522, 500, 481, 505, 519, 517, 513, 545, 548, 525, 513, 544, 539, 492, 493, 537, 498, 522, 530, 510, 551, 520, 513, 485, 496] payload_rows_per_dst=[5624, 5588, 5718, 5588, 5674, 5566, 5504, 5408] dup_rows_per_dst=[2760, 2734, 2760, 2622, 2584, 2542, 2472, 2392] combine_rows_per_src=[5624, 5588, 5718, 5588, 5674, 5566, 5504, 5408] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=38942 dup_rows=21004 dup_rows_remote=18462 dedup_ratio=0.320496 router_tokens_per_expert=[485, 517, 482, 501, 511, 493, 542, 510, 509, 522, 496, 503, 556, 512, 513, 527, 525, 492, 549, 520, 487, 463, 495, 507, 496, 485, 524, 525, 520, 511, 542, 499, 546, 519, 519, 513, 507, 506, 492, 519, 469, 504, 549, 506, 517, 526, 513, 482, 523, 526, 525, 525, 500, 496, 536, 530, 490, 522, 513, 490, 531, 512, 515, 528] payload_rows_per_dst=[5508, 5580, 5460, 5612, 5578, 5590, 5650, 5554] dup_rows_per_dst=[2574, 2696, 2616, 2592, 2664, 2542, 2672, 2648] combine_rows_per_src=[5508, 5580, 5460, 5612, 5578, 5590, 5650, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38892 dup_rows=21118 dup_rows_remote=18468 dedup_ratio=0.322235 router_tokens_per_expert=[503, 505, 540, 475, 542, 531, 494, 498, 493, 509, 517, 518, 500, 502, 479, 514, 500, 548, 531, 500, 473, 470, 532, 534, 479, 515, 492, 521, 456, 494, 529, 496, 557, 513, 497, 562, 541, 481, 468, 537, 555, 533, 567, 552, 517, 504, 497, 522, 545, 501, 500, 500, 504, 500, 495, 547, 530, 515, 484, 512, 520, 500, 478, 544] payload_rows_per_dst=[5522, 5424, 5526, 5474, 5654, 5654, 5584, 5580] dup_rows_per_dst=[2654, 2640, 2650, 2490, 2658, 2840, 2600, 2586] combine_rows_per_src=[5522, 5424, 5526, 5474, 5654, 5654, 5584, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=229248 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279179136 ingress_bytes=499150848 link_bytes=499150848 token_bytes=558129024 weight_movement_bytes=220200960 total_nvlink_bytes=778329984 weight_to_token_ratio=0.394534 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681054209e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277401600 dispatch_meta_bytes=228784 combine_payload_bytes=277401600 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277630384 ingress_bytes=607703040 link_bytes=607703040 token_bytes=555031984 weight_movement_bytes=330301440 total_nvlink_bytes=885333424 weight_to_token_ratio=0.595103 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056895537e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44314 payload_rows_remote=38724 dup_rows=21222 dup_rows_remote=18490 dedup_ratio=0.323822 router_tokens_per_expert=[503, 494, 515, 514, 506, 509, 499, 499, 480, 504, 492, 522, 504, 528, 527, 538, 522, 499, 528, 501, 524, 491, 511, 505, 533, 527, 497, 558, 484, 516, 467, 491, 495, 549, 516, 526, 487, 528, 513, 502, 470, 505, 533, 501, 499, 487, 498, 512, 543, 528, 517, 548, 509, 504, 468, 544, 523, 506, 549, 531, 502, 532, 520, 535] payload_rows_per_dst=[5566, 5526, 5556, 5578, 5502, 5406, 5590, 5590] dup_rows_per_dst=[2512, 2664, 2606, 2568, 2730, 2604, 2732, 2806] combine_rows_per_src=[5566, 5526, 5556, 5578, 5502, 5406, 5590, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38958 dup_rows=21068 dup_rows_remote=18424 dedup_ratio=0.321472 router_tokens_per_expert=[518, 479, 507, 474, 545, 537, 483, 518, 515, 528, 505, 507, 494, 498, 522, 496, 487, 538, 523, 498, 506, 502, 495, 530, 519, 530, 514, 514, 488, 472, 508, 532, 539, 506, 499, 515, 499, 536, 526, 552, 530, 511, 547, 506, 515, 482, 543, 526, 499, 531, 472, 501, 484, 527, 508, 532, 518, 489, 495, 481, 532, 522, 525, 538] payload_rows_per_dst=[5506, 5536, 5580, 5510, 5596, 5640, 5518, 5582] dup_rows_per_dst=[2616, 2594, 2578, 2644, 2748, 2680, 2590, 2618] combine_rows_per_src=[5506, 5536, 5580, 5510, 5596, 5640, 5518, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=228752 combine_payload_bytes=278691840 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278920592 ingress_bytes=608993280 link_bytes=608993280 token_bytes=557612432 weight_movement_bytes=330301440 total_nvlink_bytes=887913872 weight_to_token_ratio=0.592349 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059139476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280956928 dispatch_meta_bytes=229952 combine_payload_bytes=280956928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281186880 ingress_bytes=391057408 link_bytes=391057408 token_bytes=562143808 weight_movement_bytes=110100480 total_nvlink_bytes=672244288 weight_to_token_ratio=0.195858 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.801131504e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=228608 combine_payload_bytes=279881728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280110336 ingress_bytes=389982208 link_bytes=389982208 token_bytes=559992064 weight_movement_bytes=110100480 total_nvlink_bytes=670092544 weight_to_token_ratio=0.196611 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782432008e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=229616 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279365872 ingress_bytes=499337216 link_bytes=499337216 token_bytes=558502128 weight_movement_bytes=220200960 total_nvlink_bytes=778703088 weight_to_token_ratio=0.394271 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684295455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=229440 combine_payload_bytes=278777856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279007296 ingress_bytes=388878336 link_bytes=388878336 token_bytes=557785152 weight_movement_bytes=110100480 total_nvlink_bytes=667885632 weight_to_token_ratio=0.197389 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763233858e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=38942 dup_rows=21004 dup_rows_remote=18462 dedup_ratio=0.320496 router_tokens_per_expert=[485, 517, 482, 501, 511, 493, 542, 510, 509, 522, 496, 503, 556, 512, 513, 527, 525, 492, 549, 520, 487, 463, 495, 507, 496, 485, 524, 525, 520, 511, 542, 499, 546, 519, 519, 513, 507, 506, 492, 519, 469, 504, 549, 506, 517, 526, 513, 482, 523, 526, 525, 525, 500, 496, 536, 530, 490, 522, 513, 490, 531, 512, 515, 528] payload_rows_per_dst=[5508, 5580, 5460, 5612, 5578, 5590, 5650, 5554] dup_rows_per_dst=[2574, 2696, 2616, 2592, 2664, 2542, 2672, 2648] combine_rows_per_src=[5508, 5580, 5460, 5612, 5578, 5590, 5650, 5554] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44470 payload_rows_remote=38874 dup_rows=21066 dup_rows_remote=18418 dedup_ratio=0.321442 router_tokens_per_expert=[518, 505, 551, 491, 555, 549, 460, 500, 546, 490, 503, 548, 482, 503, 482, 531, 518, 493, 512, 510, 540, 508, 511, 530, 530, 491, 519, 501, 509, 482, 530, 528, 494, 518, 468, 568, 484, 537, 527, 499, 500, 519, 544, 521, 496, 502, 504, 536, 517, 510, 513, 498, 482, 527, 488, 526, 521, 496, 485, 500, 545, 503, 491, 523] payload_rows_per_dst=[5574, 5506, 5596, 5522, 5554, 5578, 5570, 5570] dup_rows_per_dst=[2684, 2664, 2648, 2658, 2636, 2666, 2552, 2558] combine_rows_per_src=[5574, 5506, 5596, 5522, 5554, 5578, 5570, 5570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5570045760 prefetch_bytes=990904320 reduce_bytes=1651507200 weight_movement_bytes=2642411520 egress_bytes=2786166592 ingress_bytes=5426290688 link_bytes=5426290688 total_nvlink_bytes=8212457280 weight_to_token_ratio=0.474397 prefetch_share_of_ingress=0.182612 dedup_ratio=0.321707 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=15 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.437211978e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=228856 combine_payload_bytes=277573632 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277802488 ingress_bytes=497774592 link_bytes=497774592 token_bytes=555376120 weight_movement_bytes=220200960 total_nvlink_bytes=775577080 weight_to_token_ratio=0.396490 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.657118853e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44408 payload_rows_remote=38926 dup_rows=21128 dup_rows_remote=18442 dedup_ratio=0.322388 router_tokens_per_expert=[477, 523, 487, 482, 549, 513, 508, 509, 506, 489, 513, 518, 475, 521, 552, 510, 493, 548, 520, 479, 568, 520, 505, 517, 543, 518, 479, 487, 480, 494, 502, 536, 496, 499, 496, 513, 556, 515, 553, 524, 481, 504, 520, 547, 516, 519, 502, 519, 543, 553, 504, 504, 517, 485, 534, 486, 495, 548, 530, 512, 474, 500, 502, 500] payload_rows_per_dst=[5480, 5482, 5628, 5544, 5602, 5620, 5566, 5486] dup_rows_per_dst=[2616, 2686, 2672, 2534, 2702, 2596, 2686, 2636] combine_rows_per_src=[5480, 5482, 5628, 5544, 5602, 5620, 5566, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38880 dup_rows=21026 dup_rows_remote=18308 dedup_ratio=0.320831 router_tokens_per_expert=[532, 503, 471, 515, 519, 490, 476, 499, 531, 495, 492, 526, 506, 515, 503, 524, 517, 525, 485, 540, 520, 507, 480, 523, 509, 513, 522, 534, 477, 542, 476, 538, 520, 503, 481, 513, 534, 522, 526, 575, 517, 513, 534, 515, 541, 513, 501, 511, 526, 490, 500, 526, 498, 519, 519, 522, 508, 507, 489, 480, 511, 526, 509, 514] payload_rows_per_dst=[5422, 5576, 5536, 5656, 5630, 5660, 5552, 5478] dup_rows_per_dst=[2588, 2608, 2658, 2566, 2718, 2630, 2648, 2610] combine_rows_per_src=[5422, 5576, 5536, 5656, 5630, 5660, 5552, 5478] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=229528 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279480472 ingress_bytes=389351424 link_bytes=389351424 token_bytes=558731416 weight_movement_bytes=110100480 total_nvlink_bytes=668831896 weight_to_token_ratio=0.197054 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771461636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5563819744 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=2783051488 ingress_bytes=4432275456 link_bytes=4432275456 total_nvlink_bytes=7215326944 weight_to_token_ratio=0.296830 prefetch_share_of_ingress=0.248406 dedup_ratio=0.321487 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.708456002e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=229616 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279365872 ingress_bytes=499337216 link_bytes=499337216 token_bytes=558502128 weight_movement_bytes=220200960 total_nvlink_bytes=778703088 weight_to_token_ratio=0.394271 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684295455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44388 payload_rows_remote=38700 dup_rows=21148 dup_rows_remote=18496 dedup_ratio=0.322693 router_tokens_per_expert=[536, 540, 469, 476, 509, 556, 540, 464, 525, 481, 531, 517, 489, 522, 509, 534, 511, 478, 484, 500, 481, 533, 498, 502, 494, 512, 518, 499, 469, 525, 576, 536, 511, 490, 479, 502, 514, 519, 502, 482, 518, 497, 503, 514, 528, 552, 516, 524, 536, 509, 527, 515, 477, 530, 545, 496, 508, 555, 520, 532, 505, 497, 532, 519] payload_rows_per_dst=[5562, 5484, 5436, 5594, 5398, 5574, 5652, 5688] dup_rows_per_dst=[2618, 2732, 2538, 2664, 2524, 2730, 2690, 2652] combine_rows_per_src=[5562, 5484, 5436, 5594, 5398, 5574, 5652, 5688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582725200 prefetch_bytes=660602880 reduce_bytes=990904320 weight_movement_bytes=1651507200 egress_bytes=2792509520 ingress_bytes=4441722880 link_bytes=4441722880 total_nvlink_bytes=7234232400 weight_to_token_ratio=0.295825 prefetch_share_of_ingress=0.148727 dedup_ratio=0.321011 prefetch_slots=12 unused_prefetch_slots=148 slot_utilization=0.075000 reduce_slots=9 remote_demand=6 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.724886626e-05 prefetch_bytes_per_gemm_flop=1.148897059e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=228752 combine_payload_bytes=278691840 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278920592 ingress_bytes=608993280 link_bytes=608993280 token_bytes=557612432 weight_movement_bytes=330301440 total_nvlink_bytes=887913872 weight_to_token_ratio=0.592349 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059139476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279021568 dispatch_meta_bytes=229472 combine_payload_bytes=279021568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279251040 ingress_bytes=389122048 link_bytes=389122048 token_bytes=558272608 weight_movement_bytes=110100480 total_nvlink_bytes=668373088 weight_to_token_ratio=0.197216 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767472411e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5590639264 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=2796466848 ingress_bytes=4775981056 link_bytes=4775981056 total_nvlink_bytes=7572447904 weight_to_token_ratio=0.354487 prefetch_share_of_ingress=0.230530 dedup_ratio=0.320770 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.306216570e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5583012056 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=2792653016 ingress_bytes=4882268160 link_bytes=4882268160 total_nvlink_bytes=7674921176 weight_to_token_ratio=0.374692 prefetch_share_of_ingress=0.202960 dedup_ratio=0.321088 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.491067325e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44658 payload_rows_remote=39080 dup_rows=20878 dup_rows_remote=18134 dedup_ratio=0.318573 router_tokens_per_expert=[486, 514, 460, 518, 488, 532, 527, 510, 509, 488, 538, 501, 463, 486, 544, 504, 520, 492, 561, 528, 510, 531, 519, 500, 510, 529, 535, 508, 535, 511, 523, 486, 546, 484, 529, 530, 495, 504, 541, 523, 491, 501, 507, 498, 517, 489, 482, 518, 533, 504, 502, 512, 499, 522, 484, 530, 535, 531, 492, 531, 521, 522, 525, 504] payload_rows_per_dst=[5508, 5590, 5578, 5688, 5628, 5478, 5568, 5620] dup_rows_per_dst=[2562, 2476, 2744, 2586, 2676, 2528, 2604, 2702] combine_rows_per_src=[5508, 5590, 5578, 5688, 5628, 5478, 5568, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277401600 dispatch_meta_bytes=228784 combine_payload_bytes=277401600 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277630384 ingress_bytes=607703040 link_bytes=607703040 token_bytes=555031984 weight_movement_bytes=330301440 total_nvlink_bytes=885333424 weight_to_token_ratio=0.595103 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056895537e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5583098752 prefetch_bytes=660602880 reduce_bytes=1321205760 weight_movement_bytes=1981808640 egress_bytes=2792696704 ingress_bytes=4772210688 link_bytes=4772440184 total_nvlink_bytes=7564907392 weight_to_token_ratio=0.354966 prefetch_share_of_ingress=0.138427 dedup_ratio=0.321518 prefetch_slots=12 unused_prefetch_slots=148 slot_utilization=0.075000 reduce_slots=12 remote_demand=6 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.300058412e-05 prefetch_bytes_per_gemm_flop=1.148897059e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280125440 dispatch_meta_bytes=228856 combine_payload_bytes=280125440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280354296 ingress_bytes=390225920 link_bytes=390225920 token_bytes=560479736 weight_movement_bytes=110100480 total_nvlink_bytes=670580216 weight_to_token_ratio=0.196440 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786670560e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5570045760 prefetch_bytes=990904320 reduce_bytes=1651507200 weight_movement_bytes=2642411520 egress_bytes=2786166592 ingress_bytes=5426290688 link_bytes=5426290688 total_nvlink_bytes=8212457280 weight_to_token_ratio=0.474397 prefetch_share_of_ingress=0.182612 dedup_ratio=0.321707 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=15 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.437211978e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44678 payload_rows_remote=39196 dup_rows=20858 dup_rows_remote=18292 dedup_ratio=0.318268 router_tokens_per_expert=[550, 519, 515, 536, 530, 520, 513, 528, 495, 524, 520, 494, 515, 510, 474, 492, 503, 493, 535, 526, 533, 510, 536, 489, 495, 531, 546, 531, 494, 519, 461, 504, 536, 503, 514, 497, 483, 484, 520, 463, 487, 515, 521, 505, 532, 510, 488, 506, 515, 491, 520, 515, 536, 492, 500, 503, 560, 511, 537, 524, 511, 479, 547, 522] payload_rows_per_dst=[5744, 5482, 5632, 5542, 5526, 5532, 5542, 5678] dup_rows_per_dst=[2678, 2566, 2618, 2620, 2474, 2596, 2602, 2704] combine_rows_per_src=[5744, 5482, 5632, 5542, 5526, 5532, 5542, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280956928 dispatch_meta_bytes=229952 combine_payload_bytes=280956928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281186880 ingress_bytes=391057408 link_bytes=391057408 token_bytes=562143808 weight_movement_bytes=110100480 total_nvlink_bytes=672244288 weight_to_token_ratio=0.195858 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.801131504e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44470 payload_rows_remote=38874 dup_rows=21066 dup_rows_remote=18418 dedup_ratio=0.321442 router_tokens_per_expert=[518, 505, 551, 491, 555, 549, 460, 500, 546, 490, 503, 548, 482, 503, 482, 531, 518, 493, 512, 510, 540, 508, 511, 530, 530, 491, 519, 501, 509, 482, 530, 528, 494, 518, 468, 568, 484, 537, 527, 499, 500, 519, 544, 521, 496, 502, 504, 536, 517, 510, 513, 498, 482, 527, 488, 526, 521, 496, 485, 500, 545, 503, 491, 523] payload_rows_per_dst=[5574, 5506, 5596, 5522, 5554, 5578, 5570, 5570] dup_rows_per_dst=[2684, 2664, 2648, 2658, 2636, 2666, 2552, 2558] combine_rows_per_src=[5574, 5506, 5596, 5522, 5554, 5578, 5570, 5570] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588691296 prefetch_bytes=990904320 reduce_bytes=330301440 weight_movement_bytes=1321205760 egress_bytes=2795493728 ingress_bytes=4114403328 link_bytes=4114403328 total_nvlink_bytes=6909897056 weight_to_token_ratio=0.236407 prefetch_share_of_ingress=0.240838 dedup_ratio=0.320923 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=3 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.155624090e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5583012056 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=2792653016 ingress_bytes=4882268160 link_bytes=4882268160 total_nvlink_bytes=7674921176 weight_to_token_ratio=0.374692 prefetch_share_of_ingress=0.202960 dedup_ratio=0.321088 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.491067325e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5563819744 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=2783051488 ingress_bytes=4432275456 link_bytes=4432275456 total_nvlink_bytes=7215326944 weight_to_token_ratio=0.296830 prefetch_share_of_ingress=0.248406 dedup_ratio=0.321487 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.708456002e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582725200 prefetch_bytes=660602880 reduce_bytes=990904320 weight_movement_bytes=1651507200 egress_bytes=2792509520 ingress_bytes=4441722880 link_bytes=4441722880 total_nvlink_bytes=7234232400 weight_to_token_ratio=0.295825 prefetch_share_of_ingress=0.148727 dedup_ratio=0.321011 prefetch_slots=12 unused_prefetch_slots=148 slot_utilization=0.075000 reduce_slots=9 remote_demand=6 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.724886626e-05 prefetch_bytes_per_gemm_flop=1.148897059e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=229168 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278878000 ingress_bytes=388749312 link_bytes=388749312 token_bytes=557526832 weight_movement_bytes=110100480 total_nvlink_bytes=667627312 weight_to_token_ratio=0.197480 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760989918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5590639264 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=2796466848 ingress_bytes=4775981056 link_bytes=4775981056 total_nvlink_bytes=7572447904 weight_to_token_ratio=0.354487 prefetch_share_of_ingress=0.230530 dedup_ratio=0.320770 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.306216570e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582837400 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2792564376 ingress_bytes=4661981184 link_bytes=4661981184 total_nvlink_bytes=7454545560 weight_to_token_ratio=0.335261 prefetch_share_of_ingress=0.236167 dedup_ratio=0.320651 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.107952043e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none - [2026-08-26 16:58:52] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 360.8 | throughput per GPU (TFLOP/s/GPU): 342.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.515437E+00 | loss scale: 1.0 | grad norm: 30.549 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38888 dup_rows=21026 dup_rows_remote=18380 dedup_ratio=0.320831 router_tokens_per_expert=[521, 510, 480, 520, 528, 513, 510, 506, 481, 493, 540, 556, 504, 516, 512, 510, 516, 500, 551, 566, 502, 522, 495, 480, 547, 507, 523, 496, 530, 481, 538, 490, 511, 508, 498, 506, 528, 518, 514, 551, 517, 546, 489, 548, 529, 550, 505, 494, 467, 484, 500, 530, 490, 495, 485, 497, 506, 504, 524, 493, 535, 461, 547, 494] payload_rows_per_dst=[5582, 5558, 5586, 5524, 5622, 5646, 5470, 5522] dup_rows_per_dst=[2594, 2666, 2678, 2700, 2646, 2710, 2426, 2606] combine_rows_per_src=[5582, 5558, 5586, 5524, 5622, 5646, 5470, 5522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44636 payload_rows_remote=39126 dup_rows=20900 dup_rows_remote=18358 dedup_ratio=0.318909 router_tokens_per_expert=[573, 511, 503, 499, 508, 515, 513, 508, 524, 496, 497, 531, 453, 485, 505, 535, 494, 522, 513, 539, 496, 505, 512, 558, 509, 489, 471, 495, 479, 500, 558, 534, 521, 501, 504, 502, 536, 529, 512, 517, 503, 507, 533, 514, 512, 509, 525, 500, 530, 552, 491, 525, 565, 480, 490, 522, 522, 489, 502, 511, 475, 535, 529, 495] payload_rows_per_dst=[5610, 5510, 5656, 5500, 5578, 5602, 5626, 5554] dup_rows_per_dst=[2650, 2542, 2622, 2570, 2666, 2604, 2684, 2562] combine_rows_per_src=[5610, 5510, 5656, 5500, 5578, 5602, 5626, 5554] + [2026-08-26 18:10:57] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 366.7 | throughput per GPU (TFLOP/s/GPU): 336.8 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.515504E+00 | loss scale: 1.0 | grad norm: 30.532 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38886 dup_rows=21068 dup_rows_remote=18304 dedup_ratio=0.321472 router_tokens_per_expert=[494, 534, 523, 505, 494, 491, 487, 501, 506, 522, 483, 466, 511, 504, 514, 511, 521, 504, 499, 502, 506, 519, 494, 524, 512, 516, 509, 503, 507, 556, 516, 554, 519, 493, 531, 540, 484, 518, 489, 532, 509, 551, 491, 542, 548, 486, 504, 550, 527, 536, 509, 485, 506, 504, 549, 514, 482, 488, 493, 507, 539, 514, 509, 531] payload_rows_per_dst=[5536, 5444, 5578, 5582, 5596, 5632, 5546, 5554] dup_rows_per_dst=[2522, 2590, 2560, 2764, 2616, 2730, 2714, 2572] combine_rows_per_src=[5536, 5444, 5578, 5582, 5596, 5632, 5546, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38668 dup_rows=21142 dup_rows_remote=18128 dedup_ratio=0.322601 router_tokens_per_expert=[526, 522, 534, 495, 499, 513, 517, 517, 505, 530, 516, 506, 549, 509, 522, 522, 483, 483, 544, 521, 535, 514, 485, 452, 551, 540, 511, 539, 509, 488, 493, 516, 515, 499, 531, 474, 515, 530, 515, 492, 504, 510, 507, 539, 479, 497, 525, 489, 522, 523, 525, 503, 498, 495, 520, 486, 502, 527, 514, 493, 518, 520, 544, 511] payload_rows_per_dst=[5630, 5592, 5542, 5284, 5464, 5560, 5596, 5726] dup_rows_per_dst=[2616, 2726, 2492, 2298, 2688, 2576, 2732, 3014] combine_rows_per_src=[5630, 5592, 5542, 5284, 5464, 5560, 5596, 5726] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44310 payload_rows_remote=38742 dup_rows=21226 dup_rows_remote=18492 dedup_ratio=0.323883 router_tokens_per_expert=[518, 507, 504, 511, 454, 514, 511, 512, 504, 514, 508, 483, 536, 569, 499, 472, 487, 468, 545, 519, 516, 495, 508, 498, 527, 525, 541, 531, 514, 522, 504, 547, 500, 480, 508, 522, 504, 561, 510, 494, 526, 529, 535, 532, 495, 497, 544, 493, 484, 505, 522, 515, 509, 496, 511, 521, 519, 497, 488, 497, 531, 546, 534, 500] payload_rows_per_dst=[5456, 5506, 5520, 5634, 5562, 5568, 5516, 5548] dup_rows_per_dst=[2606, 2664, 2552, 2788, 2596, 2734, 2610, 2676] combine_rows_per_src=[5456, 5506, 5520, 5634, 5562, 5568, 5516, 5548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38960 dup_rows=21042 dup_rows_remote=18460 dedup_ratio=0.321075 router_tokens_per_expert=[475, 541, 493, 504, 534, 458, 518, 490, 489, 497, 529, 483, 545, 512, 529, 527, 495, 486, 503, 506, 526, 515, 524, 481, 525, 537, 520, 542, 527, 510, 518, 526, 531, 545, 484, 497, 511, 493, 551, 516, 531, 540, 500, 453, 564, 530, 539, 525, 494, 518, 515, 492, 502, 529, 544, 464, 477, 516, 524, 516, 496, 520, 510, 476] payload_rows_per_dst=[5522, 5556, 5512, 5648, 5572, 5652, 5534, 5498] dup_rows_per_dst=[2504, 2666, 2560, 2762, 2684, 2712, 2582, 2572] combine_rows_per_src=[5522, 5556, 5512, 5648, 5572, 5652, 5534, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38888 dup_rows=21026 dup_rows_remote=18380 dedup_ratio=0.320831 router_tokens_per_expert=[521, 510, 480, 520, 528, 513, 510, 506, 481, 493, 540, 556, 504, 516, 512, 510, 516, 500, 551, 566, 502, 522, 495, 480, 547, 507, 523, 496, 530, 481, 538, 490, 511, 508, 498, 506, 528, 518, 514, 551, 517, 546, 489, 548, 529, 550, 505, 494, 467, 484, 500, 530, 490, 495, 485, 497, 506, 504, 524, 493, 535, 461, 547, 494] payload_rows_per_dst=[5582, 5558, 5586, 5524, 5622, 5646, 5470, 5522] dup_rows_per_dst=[2594, 2666, 2678, 2700, 2646, 2710, 2426, 2606] combine_rows_per_src=[5582, 5558, 5586, 5524, 5622, 5646, 5470, 5522] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38872 dup_rows=21000 dup_rows_remote=18512 dedup_ratio=0.320435 router_tokens_per_expert=[509, 486, 497, 485, 489, 515, 474, 531, 526, 514, 540, 538, 523, 484, 565, 508, 499, 545, 510, 495, 508, 464, 519, 536, 528, 531, 526, 536, 507, 497, 490, 505, 494, 514, 502, 518, 519, 494, 508, 550, 503, 559, 509, 493, 503, 466, 518, 537, 484, 480, 507, 558, 513, 541, 520, 558, 491, 511, 504, 501, 466, 493, 529, 545] payload_rows_per_dst=[5438, 5612, 5664, 5564, 5540, 5546, 5642, 5530] dup_rows_per_dst=[2534, 2784, 2488, 2676, 2658, 2630, 2680, 2550] combine_rows_per_src=[5438, 5612, 5664, 5564, 5540, 5546, 5642, 5530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44636 payload_rows_remote=39126 dup_rows=20900 dup_rows_remote=18358 dedup_ratio=0.318909 router_tokens_per_expert=[573, 511, 503, 499, 508, 515, 513, 508, 524, 496, 497, 531, 453, 485, 505, 535, 494, 522, 513, 539, 496, 505, 512, 558, 509, 489, 471, 495, 479, 500, 558, 534, 521, 501, 504, 502, 536, 529, 512, 517, 503, 507, 533, 514, 512, 509, 525, 500, 530, 552, 491, 525, 565, 480, 490, 522, 522, 489, 502, 511, 475, 535, 529, 495] payload_rows_per_dst=[5610, 5510, 5656, 5500, 5578, 5602, 5626, 5554] dup_rows_per_dst=[2650, 2542, 2622, 2570, 2666, 2604, 2684, 2562] combine_rows_per_src=[5610, 5510, 5656, 5500, 5578, 5602, 5626, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38668 dup_rows=21142 dup_rows_remote=18128 dedup_ratio=0.322601 router_tokens_per_expert=[526, 522, 534, 495, 499, 513, 517, 517, 505, 530, 516, 506, 549, 509, 522, 522, 483, 483, 544, 521, 535, 514, 485, 452, 551, 540, 511, 539, 509, 488, 493, 516, 515, 499, 531, 474, 515, 530, 515, 492, 504, 510, 507, 539, 479, 497, 525, 489, 522, 523, 525, 503, 498, 495, 520, 486, 502, 527, 514, 493, 518, 520, 544, 511] payload_rows_per_dst=[5630, 5592, 5542, 5284, 5464, 5560, 5596, 5726] dup_rows_per_dst=[2616, 2726, 2492, 2298, 2688, 2576, 2732, 3014] combine_rows_per_src=[5630, 5592, 5542, 5284, 5464, 5560, 5596, 5726] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229072 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278978256 ingress_bytes=609050624 link_bytes=609050624 token_bytes=557727440 weight_movement_bytes=330301440 total_nvlink_bytes=888028880 weight_to_token_ratio=0.592227 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059239207e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=38716 dup_rows=21060 dup_rows_remote=18084 dedup_ratio=0.321350 router_tokens_per_expert=[496, 543, 543, 485, 516, 506, 528, 521, 477, 463, 486, 511, 506, 497, 531, 511, 561, 528, 533, 512, 509, 532, 486, 477, 505, 494, 445, 525, 541, 545, 540, 526, 512, 520, 518, 502, 542, 554, 565, 514, 485, 486, 488, 491, 543, 523, 486, 545, 511, 522, 519, 491, 502, 518, 513, 513, 506, 449, 517, 488, 496, 518, 514, 538] payload_rows_per_dst=[5760, 5510, 5654, 5572, 5534, 5280, 5638, 5528] dup_rows_per_dst=[2976, 2610, 2872, 2670, 2514, 2354, 2540, 2524] combine_rows_per_src=[5760, 5510, 5654, 5572, 5534, 5280, 5638, 5528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280455168 dispatch_meta_bytes=229936 combine_payload_bytes=280455168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280685104 ingress_bytes=390555648 link_bytes=390555648 token_bytes=561140272 weight_movement_bytes=110100480 total_nvlink_bytes=671240752 weight_to_token_ratio=0.196208 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.792405072e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=38716 dup_rows=21060 dup_rows_remote=18084 dedup_ratio=0.321350 router_tokens_per_expert=[496, 543, 543, 485, 516, 506, 528, 521, 477, 463, 486, 511, 506, 497, 531, 511, 561, 528, 533, 512, 509, 532, 486, 477, 505, 494, 445, 525, 541, 545, 540, 526, 512, 520, 518, 502, 542, 554, 565, 514, 485, 486, 488, 491, 543, 523, 486, 545, 511, 522, 519, 491, 502, 518, 513, 513, 506, 449, 517, 488, 496, 518, 514, 538] payload_rows_per_dst=[5760, 5510, 5654, 5572, 5534, 5280, 5638, 5528] dup_rows_per_dst=[2976, 2610, 2872, 2670, 2514, 2354, 2540, 2524] combine_rows_per_src=[5760, 5510, 5654, 5572, 5534, 5280, 5638, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=228760 combine_payload_bytes=278734848 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278963608 ingress_bytes=719136768 link_bytes=719136768 token_bytes=557698456 weight_movement_bytes=440401920 total_nvlink_bytes=998100376 weight_to_token_ratio=0.789677 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250697117e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277172224 dispatch_meta_bytes=227184 combine_payload_bytes=277172224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277399408 ingress_bytes=387272704 link_bytes=387272704 token_bytes=554571632 weight_movement_bytes=110100480 total_nvlink_bytes=664672112 weight_to_token_ratio=0.198532 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.735309277e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277702656 dispatch_meta_bytes=228936 combine_payload_bytes=277702656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277931592 ingress_bytes=497903616 link_bytes=497903616 token_bytes=555634248 weight_movement_bytes=220200960 total_nvlink_bytes=775835208 weight_to_token_ratio=0.396306 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.659362793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=229680 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279494960 ingress_bytes=389365760 link_bytes=389365760 token_bytes=558760240 weight_movement_bytes=110100480 total_nvlink_bytes=668860720 weight_to_token_ratio=0.197044 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771710963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229072 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278978256 ingress_bytes=609050624 link_bytes=609050624 token_bytes=557727440 weight_movement_bytes=330301440 total_nvlink_bytes=888028880 weight_to_token_ratio=0.592227 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059239207e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=229536 combine_payload_bytes=278634496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278864032 ingress_bytes=388734976 link_bytes=388734976 token_bytes=557498528 weight_movement_bytes=110100480 total_nvlink_bytes=667599008 weight_to_token_ratio=0.197490 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760740592e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280455168 dispatch_meta_bytes=229936 combine_payload_bytes=280455168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280685104 ingress_bytes=390555648 link_bytes=390555648 token_bytes=561140272 weight_movement_bytes=110100480 total_nvlink_bytes=671240752 weight_to_token_ratio=0.196208 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.792405072e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277172224 dispatch_meta_bytes=227184 combine_payload_bytes=277172224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277399408 ingress_bytes=387272704 link_bytes=387272704 token_bytes=554571632 weight_movement_bytes=110100480 total_nvlink_bytes=664672112 weight_to_token_ratio=0.198532 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.735309277e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277516288 dispatch_meta_bytes=227200 combine_payload_bytes=277516288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277743488 ingress_bytes=387616768 link_bytes=387616768 token_bytes=555259776 weight_movement_bytes=110100480 total_nvlink_bytes=665360256 weight_to_token_ratio=0.198286 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.741293116e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44388 payload_rows_remote=38872 dup_rows=21148 dup_rows_remote=18412 dedup_ratio=0.322693 router_tokens_per_expert=[511, 498, 521, 529, 505, 538, 478, 558, 479, 508, 535, 503, 496, 531, 512, 486, 483, 510, 494, 545, 520, 523, 493, 499, 536, 541, 476, 509, 509, 509, 546, 521, 504, 533, 531, 513, 492, 540, 515, 498, 530, 539, 555, 485, 489, 525, 503, 480, 520, 512, 530, 508, 506, 501, 510, 487, 515, 547, 510, 526, 481, 479, 477, 525] payload_rows_per_dst=[5644, 5482, 5532, 5636, 5516, 5530, 5558, 5490] dup_rows_per_dst=[2632, 2618, 2602, 2658, 2736, 2682, 2590, 2630] combine_rows_per_src=[5644, 5482, 5532, 5636, 5516, 5530, 5558, 5490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44270 payload_rows_remote=38760 dup_rows=21266 dup_rows_remote=18656 dedup_ratio=0.324493 router_tokens_per_expert=[546, 501, 553, 497, 490, 536, 527, 501, 509, 533, 510, 487, 490, 498, 526, 507, 482, 546, 546, 496, 543, 483, 546, 515, 515, 485, 486, 543, 512, 507, 495, 530, 459, 481, 525, 523, 506, 525, 515, 523, 547, 488, 538, 530, 517, 506, 510, 508, 547, 520, 505, 491, 503, 537, 510, 531, 487, 474, 481, 517, 533, 477, 497, 516] payload_rows_per_dst=[5604, 5510, 5602, 5466, 5466, 5618, 5560, 5444] dup_rows_per_dst=[2698, 2610, 2712, 2680, 2648, 2670, 2728, 2520] combine_rows_per_src=[5604, 5510, 5602, 5466, 5466, 5618, 5560, 5444] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44180 payload_rows_remote=38548 dup_rows=21356 dup_rows_remote=18444 dedup_ratio=0.325867 router_tokens_per_expert=[546, 490, 516, 489, 503, 524, 501, 506, 499, 557, 531, 536, 484, 473, 527, 526, 527, 490, 505, 504, 513, 539, 502, 526, 466, 498, 549, 492, 508, 494, 492, 500, 510, 489, 514, 547, 536, 488, 522, 506, 536, 478, 477, 491, 500, 508, 543, 523, 479, 533, 492, 536, 518, 487, 529, 532, 511, 489, 534, 500, 541, 537, 526, 543] payload_rows_per_dst=[5436, 5524, 5602, 5480, 5486, 5508, 5512, 5632] dup_rows_per_dst=[2460, 2742, 2620, 2518, 2738, 2658, 2708, 2912] combine_rows_per_src=[5436, 5524, 5602, 5480, 5486, 5508, 5512, 5632] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=229136 combine_payload_bytes=278634496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278863632 ingress_bytes=388734976 link_bytes=388734976 token_bytes=557498128 weight_movement_bytes=110100480 total_nvlink_bytes=667598608 weight_to_token_ratio=0.197490 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760740592e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38760 dup_rows=21168 dup_rows_remote=18396 dedup_ratio=0.322998 router_tokens_per_expert=[506, 526, 491, 514, 517, 469, 521, 510, 512, 557, 519, 526, 503, 507, 477, 544, 527, 511, 500, 520, 530, 543, 551, 489, 480, 484, 509, 510, 490, 522, 479, 478, 495, 506, 496, 537, 510, 502, 520, 524, 513, 548, 470, 555, 530, 531, 522, 521, 534, 552, 451, 514, 512, 505, 527, 486, 470, 510, 511, 537, 503, 484, 528, 542] payload_rows_per_dst=[5548, 5540, 5624, 5352, 5622, 5608, 5598, 5476] dup_rows_per_dst=[2560, 2750, 2718, 2552, 2558, 2772, 2564, 2694] combine_rows_per_src=[5548, 5540, 5624, 5352, 5622, 5608, 5598, 5476] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38746 dup_rows=21150 dup_rows_remote=18390 dedup_ratio=0.322723 router_tokens_per_expert=[538, 503, 484, 494, 496, 491, 479, 514, 507, 519, 488, 526, 475, 487, 496, 535, 516, 528, 526, 522, 527, 513, 502, 509, 532, 513, 511, 533, 521, 533, 523, 534, 508, 507, 520, 496, 526, 511, 533, 510, 507, 551, 502, 469, 533, 482, 522, 529, 496, 507, 513, 521, 524, 521, 497, 492, 529, 507, 530, 529, 509, 487, 494, 531] payload_rows_per_dst=[5420, 5504, 5602, 5640, 5550, 5562, 5556, 5552] dup_rows_per_dst=[2578, 2562, 2684, 2760, 2672, 2628, 2586, 2680] combine_rows_per_src=[5420, 5504, 5602, 5640, 5550, 5562, 5556, 5552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=38946 dup_rows=21018 dup_rows_remote=18398 dedup_ratio=0.320709 router_tokens_per_expert=[513, 525, 536, 528, 511, 535, 534, 498, 522, 516, 469, 532, 509, 529, 500, 500, 508, 487, 531, 506, 527, 499, 499, 539, 488, 528, 526, 535, 527, 519, 498, 497, 528, 488, 538, 513, 483, 485, 538, 513, 510, 513, 507, 529, 470, 443, 505, 538, 511, 510, 503, 520, 514, 494, 520, 506, 515, 520, 477, 542, 506, 515, 526, 517] payload_rows_per_dst=[5580, 5504, 5572, 5532, 5578, 5524, 5572, 5656] dup_rows_per_dst=[2780, 2650, 2620, 2704, 2594, 2506, 2584, 2580] combine_rows_per_src=[5580, 5504, 5572, 5532, 5578, 5524, 5572, 5656] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44628 payload_rows_remote=38978 dup_rows=20908 dup_rows_remote=18198 dedup_ratio=0.319031 router_tokens_per_expert=[524, 532, 507, 546, 511, 526, 502, 504, 472, 487, 524, 520, 500, 518, 510, 513, 528, 500, 482, 513, 510, 542, 530, 511, 518, 540, 491, 536, 483, 467, 522, 514, 491, 538, 491, 526, 478, 516, 499, 482, 515, 521, 507, 495, 523, 491, 527, 500, 521, 499, 541, 505, 500, 519, 555, 540, 500, 504, 496, 517, 572, 527, 500, 489] payload_rows_per_dst=[5658, 5516, 5608, 5512, 5556, 5596, 5650, 5532] dup_rows_per_dst=[2646, 2572, 2624, 2630, 2486, 2562, 2710, 2678] combine_rows_per_src=[5658, 5516, 5608, 5512, 5556, 5596, 5650, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=229664 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278061344 ingress_bytes=387932160 link_bytes=387932160 token_bytes=555893024 weight_movement_bytes=110100480 total_nvlink_bytes=665993504 weight_to_token_ratio=0.198061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746778301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276312064 dispatch_meta_bytes=227968 combine_payload_bytes=276312064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276540032 ingress_bytes=386412544 link_bytes=386412544 token_bytes=552852096 weight_movement_bytes=110100480 total_nvlink_bytes=662952576 weight_to_token_ratio=0.199150 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.720349680e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=228624 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278060304 ingress_bytes=498032640 link_bytes=498032640 token_bytes=555891984 weight_movement_bytes=220200960 total_nvlink_bytes=776092944 weight_to_token_ratio=0.396122 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661606733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44388 payload_rows_remote=38872 dup_rows=21148 dup_rows_remote=18412 dedup_ratio=0.322693 router_tokens_per_expert=[511, 498, 521, 529, 505, 538, 478, 558, 479, 508, 535, 503, 496, 531, 512, 486, 483, 510, 494, 545, 520, 523, 493, 499, 536, 541, 476, 509, 509, 509, 546, 521, 504, 533, 531, 513, 492, 540, 515, 498, 530, 539, 555, 485, 489, 525, 503, 480, 520, 512, 530, 508, 506, 501, 510, 487, 515, 547, 510, 526, 481, 479, 477, 525] payload_rows_per_dst=[5644, 5482, 5532, 5636, 5516, 5530, 5558, 5490] dup_rows_per_dst=[2632, 2618, 2602, 2658, 2736, 2682, 2590, 2630] combine_rows_per_src=[5644, 5482, 5532, 5636, 5516, 5530, 5558, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=38946 dup_rows=21018 dup_rows_remote=18398 dedup_ratio=0.320709 router_tokens_per_expert=[513, 525, 536, 528, 511, 535, 534, 498, 522, 516, 469, 532, 509, 529, 500, 500, 508, 487, 531, 506, 527, 499, 499, 539, 488, 528, 526, 535, 527, 519, 498, 497, 528, 488, 538, 513, 483, 485, 538, 513, 510, 513, 507, 529, 470, 443, 505, 538, 511, 510, 503, 520, 514, 494, 520, 506, 515, 520, 477, 542, 506, 515, 526, 517] payload_rows_per_dst=[5580, 5504, 5572, 5532, 5578, 5524, 5572, 5656] dup_rows_per_dst=[2780, 2650, 2620, 2704, 2594, 2506, 2584, 2580] combine_rows_per_src=[5580, 5504, 5572, 5532, 5578, 5524, 5572, 5656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44270 payload_rows_remote=38760 dup_rows=21266 dup_rows_remote=18656 dedup_ratio=0.324493 router_tokens_per_expert=[546, 501, 553, 497, 490, 536, 527, 501, 509, 533, 510, 487, 490, 498, 526, 507, 482, 546, 546, 496, 543, 483, 546, 515, 515, 485, 486, 543, 512, 507, 495, 530, 459, 481, 525, 523, 506, 525, 515, 523, 547, 488, 538, 530, 517, 506, 510, 508, 547, 520, 505, 491, 503, 537, 510, 531, 487, 474, 481, 517, 533, 477, 497, 516] payload_rows_per_dst=[5604, 5510, 5602, 5466, 5466, 5618, 5560, 5444] dup_rows_per_dst=[2698, 2610, 2712, 2680, 2648, 2670, 2728, 2520] combine_rows_per_src=[5604, 5510, 5602, 5466, 5466, 5618, 5560, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44180 payload_rows_remote=38548 dup_rows=21356 dup_rows_remote=18444 dedup_ratio=0.325867 router_tokens_per_expert=[546, 490, 516, 489, 503, 524, 501, 506, 499, 557, 531, 536, 484, 473, 527, 526, 527, 490, 505, 504, 513, 539, 502, 526, 466, 498, 549, 492, 508, 494, 492, 500, 510, 489, 514, 547, 536, 488, 522, 506, 536, 478, 477, 491, 500, 508, 543, 523, 479, 533, 492, 536, 518, 487, 529, 532, 511, 489, 534, 500, 541, 537, 526, 543] payload_rows_per_dst=[5436, 5524, 5602, 5480, 5486, 5508, 5512, 5632] dup_rows_per_dst=[2460, 2742, 2620, 2518, 2738, 2658, 2708, 2912] combine_rows_per_src=[5436, 5524, 5602, 5480, 5486, 5508, 5512, 5632] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277731328 dispatch_meta_bytes=228544 combine_payload_bytes=277731328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277959872 ingress_bytes=387831808 link_bytes=387831808 token_bytes=555691200 weight_movement_bytes=110100480 total_nvlink_bytes=665791680 weight_to_token_ratio=0.198132 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745033015e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=228624 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278060304 ingress_bytes=498032640 link_bytes=498032640 token_bytes=555891984 weight_movement_bytes=220200960 total_nvlink_bytes=776092944 weight_to_token_ratio=0.396122 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661606733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38870 dup_rows=21052 dup_rows_remote=18374 dedup_ratio=0.321228 router_tokens_per_expert=[523, 545, 518, 505, 517, 490, 555, 493, 506, 495, 498, 509, 505, 472, 522, 517, 517, 518, 510, 497, 504, 527, 504, 532, 525, 523, 534, 518, 481, 542, 507, 495, 543, 522, 510, 551, 499, 477, 505, 474, 523, 533, 526, 539, 470, 529, 509, 510, 522, 484, 518, 500, 531, 487, 533, 539, 493, 485, 534, 535, 497, 459, 540, 487] payload_rows_per_dst=[5614, 5702, 5452, 5636, 5610, 5482, 5520, 5468] dup_rows_per_dst=[2678, 2750, 2362, 2780, 2720, 2630, 2540, 2592] combine_rows_per_src=[5614, 5702, 5452, 5636, 5610, 5482, 5520, 5468] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229376 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279394304 ingress_bytes=499365888 link_bytes=499365888 token_bytes=558559232 weight_movement_bytes=220200960 total_nvlink_bytes=778760192 weight_to_token_ratio=0.394230 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684794108e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=228704 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279623008 ingress_bytes=499595264 link_bytes=499595264 token_bytes=559017312 weight_movement_bytes=220200960 total_nvlink_bytes=779218272 weight_to_token_ratio=0.393907 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688783334e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277731328 dispatch_meta_bytes=228544 combine_payload_bytes=277731328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277959872 ingress_bytes=387831808 link_bytes=387831808 token_bytes=555691200 weight_movement_bytes=110100480 total_nvlink_bytes=665791680 weight_to_token_ratio=0.198132 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745033015e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=229136 combine_payload_bytes=278634496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278863632 ingress_bytes=388734976 link_bytes=388734976 token_bytes=557498128 weight_movement_bytes=110100480 total_nvlink_bytes=667598608 weight_to_token_ratio=0.197490 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760740592e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229376 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279394304 ingress_bytes=499365888 link_bytes=499365888 token_bytes=558559232 weight_movement_bytes=220200960 total_nvlink_bytes=778760192 weight_to_token_ratio=0.394230 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684794108e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=229664 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278061344 ingress_bytes=387932160 link_bytes=387932160 token_bytes=555893024 weight_movement_bytes=110100480 total_nvlink_bytes=665993504 weight_to_token_ratio=0.198061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746778301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44506 payload_rows_remote=38900 dup_rows=21030 dup_rows_remote=18346 dedup_ratio=0.320892 router_tokens_per_expert=[494, 504, 535, 521, 540, 482, 486, 489, 521, 511, 527, 483, 496, 463, 520, 523, 526, 498, 534, 479, 515, 510, 488, 489, 545, 506, 501, 490, 515, 556, 507, 496, 527, 537, 500, 495, 500, 537, 478, 571, 498, 498, 532, 542, 485, 481, 500, 497, 532, 550, 506, 509, 531, 501, 532, 523, 543, 530, 522, 514, 478, 533, 526, 510] payload_rows_per_dst=[5532, 5538, 5520, 5610, 5606, 5486, 5590, 5624] dup_rows_per_dst=[2570, 2550, 2558, 2622, 2684, 2580, 2778, 2688] combine_rows_per_src=[5532, 5538, 5520, 5610, 5606, 5486, 5590, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276312064 dispatch_meta_bytes=227968 combine_payload_bytes=276312064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276540032 ingress_bytes=386412544 link_bytes=386412544 token_bytes=552852096 weight_movement_bytes=110100480 total_nvlink_bytes=662952576 weight_to_token_ratio=0.199150 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.720349680e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278620160 dispatch_meta_bytes=228976 combine_payload_bytes=278620160 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278849136 ingress_bytes=719022080 link_bytes=719022080 token_bytes=557469296 weight_movement_bytes=440401920 total_nvlink_bytes=997871216 weight_to_token_ratio=0.790002 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250497656e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38914 dup_rows=21142 dup_rows_remote=18644 dedup_ratio=0.322601 router_tokens_per_expert=[518, 541, 520, 558, 512, 532, 536, 531, 441, 525, 529, 543, 528, 495, 539, 543, 515, 539, 543, 487, 519, 504, 517, 507, 522, 528, 503, 504, 511, 494, 512, 509, 500, 512, 537, 494, 517, 536, 531, 517, 495, 503, 473, 523, 508, 521, 502, 431, 515, 488, 485, 506, 495, 522, 538, 525, 517, 515, 481, 519, 541, 488, 477, 451] payload_rows_per_dst=[5564, 5518, 5494, 5562, 5636, 5496, 5644, 5480] dup_rows_per_dst=[2708, 2658, 2560, 2656, 2808, 2564, 2690, 2498] combine_rows_per_src=[5564, 5518, 5494, 5562, 5636, 5496, 5644, 5480] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=38930 dup_rows=20954 dup_rows_remote=18168 dedup_ratio=0.319733 router_tokens_per_expert=[522, 528, 529, 533, 525, 515, 490, 528, 544, 551, 529, 525, 536, 516, 499, 519, 502, 512, 510, 550, 519, 513, 532, 553, 532, 500, 522, 477, 478, 493, 507, 542, 531, 496, 511, 505, 513, 486, 488, 536, 490, 488, 515, 515, 545, 478, 495, 532, 536, 525, 496, 488, 479, 507, 505, 530, 476, 479, 475, 493, 468, 505, 527, 524] payload_rows_per_dst=[5688, 5652, 5684, 5534, 5524, 5528, 5518, 5454] dup_rows_per_dst=[2652, 2786, 2698, 2568, 2608, 2588, 2614, 2440] combine_rows_per_src=[5688, 5652, 5684, 5534, 5524, 5528, 5518, 5454] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44520 payload_rows_remote=38910 dup_rows=21016 dup_rows_remote=18400 dedup_ratio=0.320679 router_tokens_per_expert=[505, 492, 488, 494, 482, 509, 531, 486, 517, 507, 537, 501, 492, 480, 514, 538, 526, 566, 530, 527, 491, 548, 470, 488, 499, 517, 481, 518, 563, 531, 487, 507, 552, 552, 477, 490, 524, 511, 486, 484, 503, 491, 511, 515, 531, 502, 523, 537, 516, 509, 545, 522, 499, 527, 522, 537, 503, 525, 517, 529, 492, 519, 474, 521] payload_rows_per_dst=[5380, 5588, 5648, 5596, 5544, 5610, 5666, 5488] dup_rows_per_dst=[2594, 2584, 2644, 2610, 2608, 2616, 2688, 2672] combine_rows_per_src=[5380, 5588, 5648, 5596, 5544, 5610, 5666, 5488] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38870 dup_rows=21082 dup_rows_remote=18378 dedup_ratio=0.321686 router_tokens_per_expert=[506, 510, 520, 493, 496, 531, 502, 532, 493, 490, 541, 536, 525, 486, 476, 514, 526, 507, 509, 525, 497, 543, 501, 529, 515, 546, 498, 515, 516, 513, 509, 532, 518, 503, 493, 536, 515, 498, 507, 514, 535, 527, 526, 534, 514, 538, 506, 530, 504, 486, 523, 497, 482, 497, 501, 486, 538, 502, 530, 536, 482, 508, 481, 489] payload_rows_per_dst=[5570, 5490, 5582, 5584, 5516, 5688, 5452, 5572] dup_rows_per_dst=[2610, 2632, 2692, 2704, 2652, 2732, 2500, 2560] combine_rows_per_src=[5570, 5490, 5582, 5584, 5516, 5688, 5452, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44242 payload_rows_remote=38706 dup_rows=21294 dup_rows_remote=18672 dedup_ratio=0.324921 router_tokens_per_expert=[533, 527, 524, 532, 487, 532, 477, 534, 543, 489, 504, 510, 483, 493, 495, 502, 516, 478, 524, 481, 521, 533, 531, 495, 504, 516, 517, 493, 495, 523, 498, 557, 499, 499, 506, 524, 495, 502, 528, 509, 492, 549, 496, 502, 545, 508, 462, 544, 495, 525, 485, 529, 520, 517, 527, 512, 506, 541, 518, 515, 510, 541, 511, 509] payload_rows_per_dst=[5496, 5482, 5536, 5564, 5538, 5508, 5508, 5610] dup_rows_per_dst=[2796, 2556, 2622, 2642, 2586, 2688, 2712, 2692] combine_rows_per_src=[5496, 5482, 5536, 5564, 5538, 5508, 5508, 5610] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=39064 dup_rows=20946 dup_rows_remote=18444 dedup_ratio=0.319611 router_tokens_per_expert=[538, 520, 480, 521, 536, 519, 473, 501, 512, 504, 505, 520, 576, 521, 501, 502, 480, 523, 533, 492, 517, 499, 540, 499, 522, 531, 474, 538, 487, 494, 499, 487, 536, 501, 544, 467, 527, 523, 555, 502, 484, 483, 515, 525, 559, 514, 492, 494, 512, 468, 510, 512, 511, 468, 515, 518, 558, 504, 524, 522, 529, 509, 548, 495] payload_rows_per_dst=[5540, 5588, 5554, 5554, 5672, 5544, 5526, 5612] dup_rows_per_dst=[2636, 2694, 2612, 2510, 2638, 2588, 2502, 2766] combine_rows_per_src=[5540, 5588, 5554, 5554, 5672, 5544, 5526, 5612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=228984 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279064184 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557899384 weight_movement_bytes=110100480 total_nvlink_bytes=667999864 weight_to_token_ratio=0.197348 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38870 dup_rows=21082 dup_rows_remote=18378 dedup_ratio=0.321686 router_tokens_per_expert=[506, 510, 520, 493, 496, 531, 502, 532, 493, 490, 541, 536, 525, 486, 476, 514, 526, 507, 509, 525, 497, 543, 501, 529, 515, 546, 498, 515, 516, 513, 509, 532, 518, 503, 493, 536, 515, 498, 507, 514, 535, 527, 526, 534, 514, 538, 506, 530, 504, 486, 523, 497, 482, 497, 501, 486, 538, 502, 530, 536, 482, 508, 481, 489] payload_rows_per_dst=[5570, 5490, 5582, 5584, 5516, 5688, 5452, 5572] dup_rows_per_dst=[2610, 2632, 2692, 2704, 2652, 2732, 2500, 2560] combine_rows_per_src=[5570, 5490, 5582, 5584, 5516, 5688, 5452, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44506 payload_rows_remote=38900 dup_rows=21030 dup_rows_remote=18346 dedup_ratio=0.320892 router_tokens_per_expert=[494, 504, 535, 521, 540, 482, 486, 489, 521, 511, 527, 483, 496, 463, 520, 523, 526, 498, 534, 479, 515, 510, 488, 489, 545, 506, 501, 490, 515, 556, 507, 496, 527, 537, 500, 495, 500, 537, 478, 571, 498, 498, 532, 542, 485, 481, 500, 497, 532, 550, 506, 509, 531, 501, 532, 523, 543, 530, 522, 514, 478, 533, 526, 510] payload_rows_per_dst=[5532, 5538, 5520, 5610, 5606, 5486, 5590, 5624] dup_rows_per_dst=[2570, 2550, 2558, 2622, 2684, 2580, 2778, 2688] combine_rows_per_src=[5532, 5538, 5520, 5610, 5606, 5486, 5590, 5624] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=230232 combine_payload_bytes=278935552 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279165784 ingress_bytes=499136512 link_bytes=499136512 token_bytes=558101336 weight_movement_bytes=220200960 total_nvlink_bytes=778302296 weight_to_token_ratio=0.394554 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680804882e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279050240 dispatch_meta_bytes=228392 combine_payload_bytes=279050240 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279278632 ingress_bytes=499251200 link_bytes=499251200 token_bytes=558328872 weight_movement_bytes=220200960 total_nvlink_bytes=778529832 weight_to_token_ratio=0.394393 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682799495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44242 payload_rows_remote=38706 dup_rows=21294 dup_rows_remote=18672 dedup_ratio=0.324921 router_tokens_per_expert=[533, 527, 524, 532, 487, 532, 477, 534, 543, 489, 504, 510, 483, 493, 495, 502, 516, 478, 524, 481, 521, 533, 531, 495, 504, 516, 517, 493, 495, 523, 498, 557, 499, 499, 506, 524, 495, 502, 528, 509, 492, 549, 496, 502, 545, 508, 462, 544, 495, 525, 485, 529, 520, 517, 527, 512, 506, 541, 518, 515, 510, 541, 511, 509] payload_rows_per_dst=[5496, 5482, 5536, 5564, 5538, 5508, 5508, 5610] dup_rows_per_dst=[2796, 2556, 2622, 2642, 2586, 2688, 2712, 2692] combine_rows_per_src=[5496, 5482, 5536, 5564, 5538, 5508, 5508, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=38930 dup_rows=20954 dup_rows_remote=18168 dedup_ratio=0.319733 router_tokens_per_expert=[522, 528, 529, 533, 525, 515, 490, 528, 544, 551, 529, 525, 536, 516, 499, 519, 502, 512, 510, 550, 519, 513, 532, 553, 532, 500, 522, 477, 478, 493, 507, 542, 531, 496, 511, 505, 513, 486, 488, 536, 490, 488, 515, 515, 545, 478, 495, 532, 536, 525, 496, 488, 479, 507, 505, 530, 476, 479, 475, 493, 468, 505, 527, 524] payload_rows_per_dst=[5688, 5652, 5684, 5534, 5524, 5528, 5518, 5454] dup_rows_per_dst=[2652, 2786, 2698, 2568, 2608, 2588, 2614, 2440] combine_rows_per_src=[5688, 5652, 5684, 5534, 5524, 5528, 5518, 5454] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278906880 dispatch_meta_bytes=229240 combine_payload_bytes=278906880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279136120 ingress_bytes=389007360 link_bytes=389007360 token_bytes=558043000 weight_movement_bytes=110100480 total_nvlink_bytes=668143480 weight_to_token_ratio=0.197297 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765477798e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278620160 dispatch_meta_bytes=228992 combine_payload_bytes=278620160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278849152 ingress_bytes=388720640 link_bytes=388720640 token_bytes=557469312 weight_movement_bytes=110100480 total_nvlink_bytes=667569792 weight_to_token_ratio=0.197501 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760491265e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38914 dup_rows=21142 dup_rows_remote=18644 dedup_ratio=0.322601 router_tokens_per_expert=[518, 541, 520, 558, 512, 532, 536, 531, 441, 525, 529, 543, 528, 495, 539, 543, 515, 539, 543, 487, 519, 504, 517, 507, 522, 528, 503, 504, 511, 494, 512, 509, 500, 512, 537, 494, 517, 536, 531, 517, 495, 503, 473, 523, 508, 521, 502, 431, 515, 488, 485, 506, 495, 522, 538, 525, 517, 515, 481, 519, 541, 488, 477, 451] payload_rows_per_dst=[5564, 5518, 5494, 5562, 5636, 5496, 5644, 5480] dup_rows_per_dst=[2708, 2658, 2560, 2656, 2808, 2564, 2690, 2498] combine_rows_per_src=[5564, 5518, 5494, 5562, 5636, 5496, 5644, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280010752 dispatch_meta_bytes=230032 combine_payload_bytes=280010752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280240784 ingress_bytes=390111232 link_bytes=390111232 token_bytes=560251536 weight_movement_bytes=110100480 total_nvlink_bytes=670352016 weight_to_token_ratio=0.196520 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784675947e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277444608 dispatch_meta_bytes=229512 combine_payload_bytes=277444608 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277674120 ingress_bytes=607746048 link_bytes=607746048 token_bytes=555118728 weight_movement_bytes=330301440 total_nvlink_bytes=885420168 weight_to_token_ratio=0.595010 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056970335e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44360 payload_rows_remote=38772 dup_rows=21176 dup_rows_remote=18564 dedup_ratio=0.323120 router_tokens_per_expert=[534, 513, 505, 523, 489, 506, 507, 516, 504, 504, 526, 500, 523, 516, 506, 531, 492, 504, 480, 541, 505, 506, 541, 494, 509, 495, 515, 527, 523, 507, 492, 542, 508, 502, 469, 492, 478, 512, 509, 488, 506, 542, 533, 501, 493, 520, 530, 535, 509, 501, 520, 506, 543, 499, 516, 480, 531, 505, 541, 550, 513, 522, 532, 506] payload_rows_per_dst=[5588, 5530, 5478, 5560, 5454, 5562, 5580, 5608] dup_rows_per_dst=[2612, 2690, 2654, 2660, 2462, 2758, 2568, 2772] combine_rows_per_src=[5588, 5530, 5478, 5560, 5454, 5562, 5580, 5608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=38888 dup_rows=20954 dup_rows_remote=18246 dedup_ratio=0.319733 router_tokens_per_expert=[520, 522, 478, 496, 499, 480, 517, 488, 539, 485, 495, 512, 544, 508, 498, 541, 526, 505, 486, 502, 497, 501, 499, 551, 499, 522, 543, 518, 520, 494, 517, 471, 517, 500, 545, 534, 513, 551, 521, 520, 530, 476, 531, 474, 486, 508, 511, 518, 505, 500, 531, 494, 513, 495, 539, 508, 528, 506, 488, 548, 527, 537, 523, 518] payload_rows_per_dst=[5482, 5634, 5584, 5556, 5694, 5504, 5534, 5594] dup_rows_per_dst=[2518, 2610, 2550, 2612, 2708, 2564, 2636, 2756] combine_rows_per_src=[5482, 5634, 5584, 5556, 5694, 5504, 5534, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278620160 dispatch_meta_bytes=228992 combine_payload_bytes=278620160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278849152 ingress_bytes=388720640 link_bytes=388720640 token_bytes=557469312 weight_movement_bytes=110100480 total_nvlink_bytes=667569792 weight_to_token_ratio=0.197501 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760491265e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=228984 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279064184 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557899384 weight_movement_bytes=110100480 total_nvlink_bytes=667999864 weight_to_token_ratio=0.197348 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44350 payload_rows_remote=38760 dup_rows=21186 dup_rows_remote=18374 dedup_ratio=0.323273 router_tokens_per_expert=[509, 530, 482, 538, 499, 474, 533, 518, 496, 513, 523, 505, 510, 503, 509, 501, 494, 491, 520, 501, 529, 544, 526, 522, 480, 524, 486, 502, 459, 517, 481, 484, 527, 511, 519, 523, 504, 546, 501, 510, 556, 483, 532, 508, 556, 475, 533, 466, 519, 543, 527, 532, 470, 538, 531, 485, 514, 530, 521, 536, 513, 512, 506, 538] payload_rows_per_dst=[5382, 5452, 5830, 5458, 5532, 5550, 5556, 5590] dup_rows_per_dst=[2250, 2606, 2958, 2408, 2750, 2668, 2734, 2812] combine_rows_per_src=[5382, 5452, 5830, 5458, 5532, 5550, 5556, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38826 dup_rows=21146 dup_rows_remote=18470 dedup_ratio=0.322662 router_tokens_per_expert=[554, 501, 495, 517, 516, 511, 547, 489, 509, 480, 536, 554, 541, 484, 509, 507, 501, 507, 498, 542, 509, 523, 487, 489, 509, 536, 501, 525, 485, 487, 483, 490, 504, 524, 516, 553, 462, 491, 492, 488, 514, 512, 509, 513, 472, 493, 521, 557, 522, 542, 515, 512, 509, 529, 548, 497, 551, 511, 496, 493, 529, 513, 543, 515] payload_rows_per_dst=[5538, 5564, 5510, 5454, 5520, 5568, 5624, 5612] dup_rows_per_dst=[2722, 2676, 2602, 2578, 2540, 2614, 2724, 2690] combine_rows_per_src=[5538, 5564, 5510, 5454, 5520, 5568, 5624, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277444608 dispatch_meta_bytes=229512 combine_payload_bytes=277444608 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277674120 ingress_bytes=607746048 link_bytes=607746048 token_bytes=555118728 weight_movement_bytes=330301440 total_nvlink_bytes=885420168 weight_to_token_ratio=0.595010 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.056970335e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279050240 dispatch_meta_bytes=228392 combine_payload_bytes=279050240 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279278632 ingress_bytes=499251200 link_bytes=499251200 token_bytes=558328872 weight_movement_bytes=220200960 total_nvlink_bytes=778529832 weight_to_token_ratio=0.394393 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682799495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=230232 combine_payload_bytes=278935552 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279165784 ingress_bytes=499136512 link_bytes=499136512 token_bytes=558101336 weight_movement_bytes=220200960 total_nvlink_bytes=778302296 weight_to_token_ratio=0.394554 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680804882e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=229344 combine_payload_bytes=277917696 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278147040 ingress_bytes=608219136 link_bytes=608219136 token_bytes=556064736 weight_movement_bytes=330301440 total_nvlink_bytes=886366176 weight_to_token_ratio=0.593998 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057793112e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=39050 dup_rows=21008 dup_rows_remote=18416 dedup_ratio=0.320557 router_tokens_per_expert=[534, 529, 526, 535, 538, 522, 519, 498, 499, 492, 525, 505, 499, 527, 511, 521, 504, 544, 528, 532, 496, 487, 533, 500, 483, 499, 522, 485, 469, 531, 526, 554, 532, 505, 552, 513, 545, 521, 482, 533, 516, 507, 508, 497, 473, 513, 504, 517, 522, 490, 483, 522, 486, 464, 506, 563, 519, 521, 504, 478, 498, 475, 541, 505] payload_rows_per_dst=[5652, 5574, 5566, 5528, 5634, 5478, 5532, 5564] dup_rows_per_dst=[2750, 2584, 2682, 2610, 2732, 2592, 2540, 2518] combine_rows_per_src=[5652, 5574, 5566, 5528, 5634, 5478, 5532, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38884 dup_rows=21052 dup_rows_remote=18540 dedup_ratio=0.321228 router_tokens_per_expert=[504, 551, 559, 513, 515, 504, 495, 488, 547, 524, 514, 510, 471, 494, 517, 551, 538, 472, 468, 526, 556, 476, 506, 489, 530, 479, 511, 550, 489, 470, 500, 527, 475, 491, 519, 504, 532, 506, 503, 537, 508, 534, 524, 520, 522, 532, 460, 503, 534, 522, 512, 517, 534, 490, 532, 492, 532, 494, 512, 492, 512, 518, 550, 511] payload_rows_per_dst=[5598, 5578, 5516, 5600, 5468, 5552, 5596, 5576] dup_rows_per_dst=[2660, 2678, 2546, 2512, 2666, 2654, 2670, 2666] combine_rows_per_src=[5598, 5578, 5516, 5600, 5468, 5552, 5596, 5576] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44222 payload_rows_remote=38738 dup_rows=21314 dup_rows_remote=18788 dedup_ratio=0.325226 router_tokens_per_expert=[519, 531, 538, 512, 524, 535, 503, 535, 486, 530, 506, 485, 549, 566, 538, 511, 493, 476, 502, 484, 471, 533, 468, 534, 497, 517, 496, 493, 549, 522, 509, 506, 504, 477, 497, 499, 524, 506, 523, 530, 544, 483, 508, 538, 522, 498, 484, 572, 476, 531, 532, 505, 487, 460, 521, 493, 526, 500, 477, 533, 480, 553, 525, 542] payload_rows_per_dst=[5656, 5584, 5456, 5500, 5480, 5542, 5484, 5520] dup_rows_per_dst=[2738, 2758, 2466, 2678, 2640, 2756, 2526, 2752] combine_rows_per_src=[5656, 5584, 5456, 5500, 5480, 5542, 5484, 5520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44400 payload_rows_remote=39004 dup_rows=21136 dup_rows_remote=18464 dedup_ratio=0.322510 router_tokens_per_expert=[486, 498, 502, 517, 511, 518, 500, 512, 520, 494, 501, 502, 510, 509, 487, 486, 543, 471, 460, 516, 514, 523, 498, 509, 511, 499, 516, 517, 502, 543, 524, 498, 526, 519, 504, 543, 503, 516, 526, 521, 527, 516, 551, 567, 529, 495, 467, 549, 515, 503, 529, 496, 534, 541, 488, 535, 522, 499, 526, 476, 515, 522, 515, 496] payload_rows_per_dst=[5506, 5506, 5396, 5558, 5608, 5612, 5672, 5542] dup_rows_per_dst=[2582, 2512, 2672, 2662, 2708, 2790, 2610, 2600] combine_rows_per_src=[5506, 5506, 5396, 5558, 5608, 5612, 5672, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=228536 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278977720 ingress_bytes=498950144 link_bytes=498950144 token_bytes=557726904 weight_movement_bytes=220200960 total_nvlink_bytes=777927864 weight_to_token_ratio=0.394819 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677563636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38884 dup_rows=21052 dup_rows_remote=18540 dedup_ratio=0.321228 router_tokens_per_expert=[504, 551, 559, 513, 515, 504, 495, 488, 547, 524, 514, 510, 471, 494, 517, 551, 538, 472, 468, 526, 556, 476, 506, 489, 530, 479, 511, 550, 489, 470, 500, 527, 475, 491, 519, 504, 532, 506, 503, 537, 508, 534, 524, 520, 522, 532, 460, 503, 534, 522, 512, 517, 534, 490, 532, 492, 532, 494, 512, 492, 512, 518, 550, 511] payload_rows_per_dst=[5598, 5578, 5516, 5600, 5468, 5552, 5596, 5576] dup_rows_per_dst=[2660, 2678, 2546, 2512, 2666, 2654, 2670, 2666] combine_rows_per_src=[5598, 5578, 5516, 5600, 5468, 5552, 5596, 5576] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=228536 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278060216 ingress_bytes=498032640 link_bytes=498032640 token_bytes=555891896 weight_movement_bytes=220200960 total_nvlink_bytes=776092856 weight_to_token_ratio=0.396122 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661606733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229184 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278533952 ingress_bytes=498505728 link_bytes=498505728 token_bytes=556838720 weight_movement_bytes=220200960 total_nvlink_bytes=777039680 weight_to_token_ratio=0.395448 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669834511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=38888 dup_rows=20954 dup_rows_remote=18246 dedup_ratio=0.319733 router_tokens_per_expert=[520, 522, 478, 496, 499, 480, 517, 488, 539, 485, 495, 512, 544, 508, 498, 541, 526, 505, 486, 502, 497, 501, 499, 551, 499, 522, 543, 518, 520, 494, 517, 471, 517, 500, 545, 534, 513, 551, 521, 520, 530, 476, 531, 474, 486, 508, 511, 518, 505, 500, 531, 494, 513, 495, 539, 508, 528, 506, 488, 548, 527, 537, 523, 518] payload_rows_per_dst=[5482, 5634, 5584, 5556, 5694, 5504, 5534, 5594] dup_rows_per_dst=[2518, 2610, 2550, 2612, 2708, 2564, 2636, 2756] combine_rows_per_src=[5482, 5634, 5584, 5556, 5694, 5504, 5534, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44400 payload_rows_remote=39004 dup_rows=21136 dup_rows_remote=18464 dedup_ratio=0.322510 router_tokens_per_expert=[486, 498, 502, 517, 511, 518, 500, 512, 520, 494, 501, 502, 510, 509, 487, 486, 543, 471, 460, 516, 514, 523, 498, 509, 511, 499, 516, 517, 502, 543, 524, 498, 526, 519, 504, 543, 503, 516, 526, 521, 527, 516, 551, 567, 529, 495, 467, 549, 515, 503, 529, 496, 534, 541, 488, 535, 522, 499, 526, 476, 515, 522, 515, 496] payload_rows_per_dst=[5506, 5506, 5396, 5558, 5608, 5612, 5672, 5542] dup_rows_per_dst=[2582, 2512, 2672, 2662, 2708, 2790, 2610, 2600] combine_rows_per_src=[5506, 5506, 5396, 5558, 5608, 5612, 5672, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38826 dup_rows=21146 dup_rows_remote=18470 dedup_ratio=0.322662 router_tokens_per_expert=[554, 501, 495, 517, 516, 511, 547, 489, 509, 480, 536, 554, 541, 484, 509, 507, 501, 507, 498, 542, 509, 523, 487, 489, 509, 536, 501, 525, 485, 487, 483, 490, 504, 524, 516, 553, 462, 491, 492, 488, 514, 512, 509, 513, 472, 493, 521, 557, 522, 542, 515, 512, 509, 529, 548, 497, 551, 511, 496, 493, 529, 513, 543, 515] payload_rows_per_dst=[5538, 5564, 5510, 5454, 5520, 5568, 5624, 5612] dup_rows_per_dst=[2722, 2676, 2602, 2578, 2540, 2614, 2724, 2690] combine_rows_per_src=[5538, 5564, 5510, 5454, 5520, 5568, 5624, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279910400 dispatch_meta_bytes=229864 combine_payload_bytes=279910400 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280140264 ingress_bytes=500111360 link_bytes=500111360 token_bytes=560050664 weight_movement_bytes=220200960 total_nvlink_bytes=780251624 weight_to_token_ratio=0.393180 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697759092e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229696 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278950208 ingress_bytes=388820992 link_bytes=388820992 token_bytes=557670720 weight_movement_bytes=110100480 total_nvlink_bytes=667771200 weight_to_token_ratio=0.197429 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762236552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44350 payload_rows_remote=38760 dup_rows=21186 dup_rows_remote=18374 dedup_ratio=0.323273 router_tokens_per_expert=[509, 530, 482, 538, 499, 474, 533, 518, 496, 513, 523, 505, 510, 503, 509, 501, 494, 491, 520, 501, 529, 544, 526, 522, 480, 524, 486, 502, 459, 517, 481, 484, 527, 511, 519, 523, 504, 546, 501, 510, 556, 483, 532, 508, 556, 475, 533, 466, 519, 543, 527, 532, 470, 538, 531, 485, 514, 530, 521, 536, 513, 512, 506, 538] payload_rows_per_dst=[5382, 5452, 5830, 5458, 5532, 5550, 5556, 5590] dup_rows_per_dst=[2250, 2606, 2958, 2408, 2750, 2668, 2734, 2812] combine_rows_per_src=[5382, 5452, 5830, 5458, 5532, 5550, 5556, 5590] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277673984 dispatch_meta_bytes=230104 combine_payload_bytes=277673984 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277904088 ingress_bytes=497874944 link_bytes=497874944 token_bytes=555578072 weight_movement_bytes=220200960 total_nvlink_bytes=775779032 weight_to_token_ratio=0.396346 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.658864140e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279580672 dispatch_meta_bytes=229872 combine_payload_bytes=279580672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279810544 ingress_bytes=389681152 link_bytes=389681152 token_bytes=559391216 weight_movement_bytes=110100480 total_nvlink_bytes=669491696 weight_to_token_ratio=0.196822 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777196149e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=38856 dup_rows=21060 dup_rows_remote=18178 dedup_ratio=0.321350 router_tokens_per_expert=[496, 525, 536, 498, 495, 523, 513, 530, 502, 519, 517, 510, 537, 510, 499, 514, 457, 482, 550, 505, 560, 523, 507, 517, 501, 522, 559, 486, 505, 512, 548, 486, 503, 518, 521, 498, 519, 516, 519, 548, 517, 461, 544, 541, 532, 493, 528, 500, 498, 503, 536, 512, 500, 512, 490, 508, 492, 505, 503, 493, 497, 508, 519, 490] payload_rows_per_dst=[5620, 5594, 5546, 5836, 5484, 5538, 5480, 5378] dup_rows_per_dst=[2882, 2626, 2656, 2986, 2372, 2554, 2618, 2366] combine_rows_per_src=[5620, 5594, 5546, 5836, 5484, 5538, 5480, 5378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38884 dup_rows=21118 dup_rows_remote=18446 dedup_ratio=0.322235 router_tokens_per_expert=[512, 509, 539, 542, 519, 512, 543, 519, 510, 481, 527, 529, 479, 510, 541, 505, 477, 508, 510, 509, 534, 488, 513, 479, 497, 501, 516, 489, 537, 500, 493, 491, 524, 514, 521, 504, 531, 534, 472, 503, 511, 507, 506, 504, 507, 533, 550, 550, 516, 511, 503, 503, 493, 544, 514, 475, 498, 502, 544, 498, 532, 523, 531, 491] payload_rows_per_dst=[5648, 5524, 5594, 5448, 5534, 5592, 5556, 5522] dup_rows_per_dst=[2742, 2640, 2442, 2600, 2672, 2744, 2562, 2716] combine_rows_per_src=[5648, 5524, 5594, 5448, 5534, 5592, 5556, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229696 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278950208 ingress_bytes=388820992 link_bytes=388820992 token_bytes=557670720 weight_movement_bytes=110100480 total_nvlink_bytes=667771200 weight_to_token_ratio=0.197429 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762236552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38998 dup_rows=21120 dup_rows_remote=18586 dedup_ratio=0.322266 router_tokens_per_expert=[474, 507, 536, 554, 520, 513, 477, 538, 517, 495, 535, 500, 511, 484, 516, 503, 488, 478, 505, 551, 507, 525, 510, 517, 480, 548, 515, 512, 544, 488, 562, 510, 555, 512, 523, 492, 524, 504, 542, 524, 495, 530, 521, 513, 491, 520, 513, 495, 508, 527, 529, 508, 520, 517, 491, 518, 524, 505, 458, 492, 498, 481, 526, 492] payload_rows_per_dst=[5372, 5482, 5572, 5640, 5600, 5584, 5748, 5418] dup_rows_per_dst=[2422, 2510, 2590, 2746, 2814, 2572, 2932, 2534] combine_rows_per_src=[5372, 5482, 5572, 5640, 5600, 5584, 5748, 5418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44626 payload_rows_remote=38960 dup_rows=20910 dup_rows_remote=18172 dedup_ratio=0.319061 router_tokens_per_expert=[476, 503, 562, 488, 519, 493, 519, 549, 516, 557, 515, 556, 503, 498, 517, 540, 503, 517, 505, 503, 536, 447, 483, 506, 546, 485, 500, 508, 509, 514, 484, 468, 501, 527, 518, 520, 551, 495, 544, 559, 508, 525, 504, 529, 531, 474, 488, 510, 479, 504, 527, 553, 518, 497, 506, 478, 510, 508, 490, 517, 516, 508, 506, 542] payload_rows_per_dst=[5576, 5666, 5538, 5518, 5650, 5586, 5556, 5536] dup_rows_per_dst=[2642, 2738, 2462, 2510, 2780, 2552, 2568, 2658] combine_rows_per_src=[5576, 5666, 5538, 5518, 5650, 5586, 5556, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=228536 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278977720 ingress_bytes=498950144 link_bytes=498950144 token_bytes=557726904 weight_movement_bytes=220200960 total_nvlink_bytes=777927864 weight_to_token_ratio=0.394819 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677563636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279580672 dispatch_meta_bytes=229872 combine_payload_bytes=279580672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279810544 ingress_bytes=389681152 link_bytes=389681152 token_bytes=559391216 weight_movement_bytes=110100480 total_nvlink_bytes=669491696 weight_to_token_ratio=0.196822 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777196149e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229184 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278533952 ingress_bytes=498505728 link_bytes=498505728 token_bytes=556838720 weight_movement_bytes=220200960 total_nvlink_bytes=777039680 weight_to_token_ratio=0.395448 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669834511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=228536 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278060216 ingress_bytes=498032640 link_bytes=498032640 token_bytes=555891896 weight_movement_bytes=220200960 total_nvlink_bytes=776092856 weight_to_token_ratio=0.396122 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661606733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278519808 dispatch_meta_bytes=228136 combine_payload_bytes=278519808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278747944 ingress_bytes=498720768 link_bytes=498720768 token_bytes=557267752 weight_movement_bytes=220200960 total_nvlink_bytes=777468712 weight_to_token_ratio=0.395144 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673574410e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38972 dup_rows=20988 dup_rows_remote=18390 dedup_ratio=0.320251 router_tokens_per_expert=[526, 509, 516, 567, 521, 521, 493, 507, 506, 535, 465, 515, 532, 517, 520, 525, 474, 532, 520, 457, 501, 533, 514, 448, 500, 515, 532, 474, 529, 536, 497, 504, 533, 463, 552, 523, 505, 494, 492, 508, 491, 541, 503, 507, 501, 522, 495, 496, 498, 538, 558, 519, 520, 494, 521, 490, 495, 487, 545, 522, 524, 521, 514, 555] payload_rows_per_dst=[5592, 5612, 5456, 5576, 5604, 5516, 5598, 5594] dup_rows_per_dst=[2728, 2618, 2502, 2598, 2536, 2596, 2678, 2732] combine_rows_per_src=[5592, 5612, 5456, 5576, 5604, 5516, 5598, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44668 payload_rows_remote=39040 dup_rows=20868 dup_rows_remote=18236 dedup_ratio=0.318420 router_tokens_per_expert=[492, 477, 518, 494, 505, 535, 497, 499, 495, 553, 459, 469, 511, 536, 532, 490, 530, 544, 498, 522, 520, 524, 501, 488, 521, 531, 521, 524, 545, 504, 508, 559, 516, 518, 510, 516, 512, 518, 510, 524, 509, 519, 476, 529, 497, 497, 570, 533, 495, 515, 461, 515, 514, 500, 477, 512, 505, 567, 540, 518, 505, 480, 471, 537] payload_rows_per_dst=[5482, 5564, 5614, 5686, 5594, 5628, 5468, 5632] dup_rows_per_dst=[2552, 2526, 2640, 2740, 2654, 2632, 2510, 2614] combine_rows_per_src=[5482, 5564, 5614, 5686, 5594, 5628, 5468, 5632] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38858 dup_rows=21058 dup_rows_remote=18346 dedup_ratio=0.321320 router_tokens_per_expert=[545, 529, 549, 552, 496, 527, 505, 541, 494, 557, 510, 514, 514, 526, 486, 511, 504, 535, 527, 539, 496, 484, 506, 508, 541, 519, 508, 475, 494, 491, 507, 510, 555, 543, 514, 527, 478, 502, 468, 509, 508, 482, 524, 481, 480, 489, 523, 473, 523, 543, 524, 516, 522, 510, 506, 522, 482, 486, 508, 502, 494, 537, 534, 503] payload_rows_per_dst=[5648, 5586, 5576, 5518, 5554, 5468, 5620, 5508] dup_rows_per_dst=[2840, 2638, 2622, 2572, 2638, 2452, 2712, 2584] combine_rows_per_src=[5648, 5586, 5576, 5518, 5554, 5468, 5620, 5508] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=39028 dup_rows=20926 dup_rows_remote=18284 dedup_ratio=0.319305 router_tokens_per_expert=[483, 485, 530, 535, 513, 477, 495, 486, 501, 544, 516, 513, 518, 493, 502, 511, 534, 514, 563, 496, 494, 499, 509, 503, 521, 490, 523, 492, 523, 540, 492, 519, 507, 545, 524, 506, 489, 538, 532, 516, 517, 528, 521, 511, 542, 515, 555, 518, 498, 516, 491, 509, 494, 496, 486, 538, 515, 499, 515, 491, 524, 509, 521, 488] payload_rows_per_dst=[5470, 5574, 5582, 5620, 5614, 5666, 5530, 5554] dup_rows_per_dst=[2538, 2622, 2642, 2580, 2700, 2748, 2526, 2570] combine_rows_per_src=[5470, 5574, 5582, 5620, 5614, 5666, 5530, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229320 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278949832 ingress_bytes=498921472 link_bytes=498921472 token_bytes=557670344 weight_movement_bytes=220200960 total_nvlink_bytes=777871304 weight_to_token_ratio=0.394859 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677064983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=230336 combine_payload_bytes=279537664 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279768000 ingress_bytes=499738624 link_bytes=499738624 token_bytes=559305664 weight_movement_bytes=220200960 total_nvlink_bytes=779506624 weight_to_token_ratio=0.393704 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.691276600e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38972 dup_rows=20988 dup_rows_remote=18390 dedup_ratio=0.320251 router_tokens_per_expert=[526, 509, 516, 567, 521, 521, 493, 507, 506, 535, 465, 515, 532, 517, 520, 525, 474, 532, 520, 457, 501, 533, 514, 448, 500, 515, 532, 474, 529, 536, 497, 504, 533, 463, 552, 523, 505, 494, 492, 508, 491, 541, 503, 507, 501, 522, 495, 496, 498, 538, 558, 519, 520, 494, 521, 490, 495, 487, 545, 522, 524, 521, 514, 555] payload_rows_per_dst=[5592, 5612, 5456, 5576, 5604, 5516, 5598, 5594] dup_rows_per_dst=[2728, 2618, 2502, 2598, 2536, 2596, 2678, 2732] combine_rows_per_src=[5592, 5612, 5456, 5576, 5604, 5516, 5598, 5594] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=228528 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279493808 ingress_bytes=719667200 link_bytes=719667200 token_bytes=558759088 weight_movement_bytes=440401920 total_nvlink_bytes=999161008 weight_to_token_ratio=0.788179 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251619626e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=229448 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279580744 ingress_bytes=389451776 link_bytes=389451776 token_bytes=558932040 weight_movement_bytes=110100480 total_nvlink_bytes=669032520 weight_to_token_ratio=0.196984 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773206923e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38884 dup_rows=21118 dup_rows_remote=18446 dedup_ratio=0.322235 router_tokens_per_expert=[512, 509, 539, 542, 519, 512, 543, 519, 510, 481, 527, 529, 479, 510, 541, 505, 477, 508, 510, 509, 534, 488, 513, 479, 497, 501, 516, 489, 537, 500, 493, 491, 524, 514, 521, 504, 531, 534, 472, 503, 511, 507, 506, 504, 507, 533, 550, 550, 516, 511, 503, 503, 493, 544, 514, 475, 498, 502, 544, 498, 532, 523, 531, 491] payload_rows_per_dst=[5648, 5524, 5594, 5448, 5534, 5592, 5556, 5522] dup_rows_per_dst=[2742, 2640, 2442, 2600, 2672, 2744, 2562, 2716] combine_rows_per_src=[5648, 5524, 5594, 5448, 5534, 5592, 5556, 5522] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229104 combine_payload_bytes=279838720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280067824 ingress_bytes=389939200 link_bytes=389939200 token_bytes=559906544 weight_movement_bytes=110100480 total_nvlink_bytes=670007024 weight_to_token_ratio=0.196641 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781684028e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44626 payload_rows_remote=38960 dup_rows=20910 dup_rows_remote=18172 dedup_ratio=0.319061 router_tokens_per_expert=[476, 503, 562, 488, 519, 493, 519, 549, 516, 557, 515, 556, 503, 498, 517, 540, 503, 517, 505, 503, 536, 447, 483, 506, 546, 485, 500, 508, 509, 514, 484, 468, 501, 527, 518, 520, 551, 495, 544, 559, 508, 525, 504, 529, 531, 474, 488, 510, 479, 504, 527, 553, 518, 497, 506, 478, 510, 508, 490, 517, 516, 508, 506, 542] payload_rows_per_dst=[5576, 5666, 5538, 5518, 5650, 5586, 5556, 5536] dup_rows_per_dst=[2642, 2738, 2462, 2510, 2780, 2552, 2568, 2658] combine_rows_per_src=[5576, 5666, 5538, 5518, 5650, 5586, 5556, 5536] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=228816 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278762960 ingress_bytes=388634624 link_bytes=388634624 token_bytes=557297104 weight_movement_bytes=110100480 total_nvlink_bytes=667397584 weight_to_token_ratio=0.197562 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758995305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=229248 combine_payload_bytes=279752704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279981952 ingress_bytes=389853184 link_bytes=389853184 token_bytes=559734656 weight_movement_bytes=110100480 total_nvlink_bytes=669835136 weight_to_token_ratio=0.196701 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780188068e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=39028 dup_rows=20926 dup_rows_remote=18284 dedup_ratio=0.319305 router_tokens_per_expert=[483, 485, 530, 535, 513, 477, 495, 486, 501, 544, 516, 513, 518, 493, 502, 511, 534, 514, 563, 496, 494, 499, 509, 503, 521, 490, 523, 492, 523, 540, 492, 519, 507, 545, 524, 506, 489, 538, 532, 516, 517, 528, 521, 511, 542, 515, 555, 518, 498, 516, 491, 509, 494, 496, 486, 538, 515, 499, 515, 491, 524, 509, 521, 488] payload_rows_per_dst=[5470, 5574, 5582, 5620, 5614, 5666, 5530, 5554] dup_rows_per_dst=[2538, 2622, 2642, 2580, 2700, 2748, 2526, 2570] combine_rows_per_src=[5470, 5574, 5582, 5620, 5614, 5666, 5530, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38998 dup_rows=21120 dup_rows_remote=18586 dedup_ratio=0.322266 router_tokens_per_expert=[474, 507, 536, 554, 520, 513, 477, 538, 517, 495, 535, 500, 511, 484, 516, 503, 488, 478, 505, 551, 507, 525, 510, 517, 480, 548, 515, 512, 544, 488, 562, 510, 555, 512, 523, 492, 524, 504, 542, 524, 495, 530, 521, 513, 491, 520, 513, 495, 508, 527, 529, 508, 520, 517, 491, 518, 524, 505, 458, 492, 498, 481, 526, 492] payload_rows_per_dst=[5372, 5482, 5572, 5640, 5600, 5584, 5748, 5418] dup_rows_per_dst=[2422, 2510, 2590, 2746, 2814, 2572, 2932, 2534] combine_rows_per_src=[5372, 5482, 5572, 5640, 5600, 5584, 5748, 5418] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44280 payload_rows_remote=38692 dup_rows=21256 dup_rows_remote=18560 dedup_ratio=0.324341 router_tokens_per_expert=[536, 524, 488, 530, 505, 531, 495, 473, 513, 523, 527, 506, 471, 557, 550, 529, 498, 490, 509, 513, 525, 540, 488, 520, 499, 498, 529, 514, 494, 501, 525, 532, 505, 535, 518, 479, 532, 495, 530, 471, 525, 504, 503, 524, 527, 489, 510, 511, 500, 485, 503, 494, 509, 471, 521, 503, 509, 551, 535, 483, 511, 522, 556, 524] payload_rows_per_dst=[5588, 5612, 5714, 5578, 5306, 5544, 5442, 5496] dup_rows_per_dst=[2696, 2624, 2990, 2606, 2286, 2656, 2530, 2868] combine_rows_per_src=[5588, 5612, 5714, 5578, 5306, 5544, 5442, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=229448 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279580744 ingress_bytes=389451776 link_bytes=389451776 token_bytes=558932040 weight_movement_bytes=110100480 total_nvlink_bytes=669032520 weight_to_token_ratio=0.196984 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773206923e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=38798 dup_rows=21110 dup_rows_remote=18344 dedup_ratio=0.322113 router_tokens_per_expert=[506, 482, 531, 498, 545, 519, 487, 504, 489, 500, 545, 491, 500, 533, 513, 510, 521, 521, 509, 505, 531, 491, 543, 531, 511, 497, 511, 518, 512, 493, 481, 505, 531, 507, 547, 537, 523, 539, 520, 493, 502, 485, 511, 532, 486, 518, 520, 489, 504, 536, 516, 524, 533, 548, 510, 514, 491, 557, 495, 516, 474, 464, 527, 486] payload_rows_per_dst=[5532, 5484, 5592, 5478, 5628, 5602, 5658, 5452] dup_rows_per_dst=[2612, 2678, 2712, 2578, 2766, 2484, 2712, 2568] combine_rows_per_src=[5532, 5484, 5592, 5478, 5628, 5602, 5658, 5452] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38822 dup_rows=21094 dup_rows_remote=18298 dedup_ratio=0.321869 router_tokens_per_expert=[504, 520, 504, 512, 485, 493, 514, 542, 497, 536, 509, 511, 561, 489, 503, 493, 526, 473, 534, 506, 536, 489, 522, 487, 486, 514, 514, 545, 522, 508, 527, 538, 485, 486, 534, 483, 513, 486, 531, 519, 494, 488, 502, 476, 544, 544, 530, 522, 524, 499, 522, 497, 490, 526, 527, 509, 528, 534, 517, 542, 523, 482, 492, 519] payload_rows_per_dst=[5532, 5568, 5542, 5554, 5502, 5528, 5596, 5620] dup_rows_per_dst=[2502, 2736, 2604, 2612, 2572, 2678, 2594, 2796] combine_rows_per_src=[5532, 5568, 5542, 5554, 5502, 5528, 5596, 5620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44714 payload_rows_remote=39108 dup_rows=20822 dup_rows_remote=18296 dedup_ratio=0.317719 router_tokens_per_expert=[488, 512, 518, 562, 540, 518, 504, 523, 539, 515, 488, 490, 492, 512, 503, 527, 546, 491, 503, 500, 526, 509, 533, 546, 535, 526, 552, 531, 494, 565, 527, 475, 529, 499, 528, 466, 527, 511, 493, 508, 539, 485, 504, 484, 478, 499, 512, 532, 476, 488, 493, 499, 527, 527, 519, 519, 479, 503, 492, 488, 513, 502, 509, 550] payload_rows_per_dst=[5654, 5606, 5652, 5642, 5498, 5558, 5538, 5566] dup_rows_per_dst=[2676, 2526, 2656, 2768, 2624, 2508, 2558, 2506] combine_rows_per_src=[5654, 5606, 5652, 5642, 5498, 5558, 5538, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229320 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278949832 ingress_bytes=498921472 link_bytes=498921472 token_bytes=557670344 weight_movement_bytes=220200960 total_nvlink_bytes=777871304 weight_to_token_ratio=0.394859 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677064983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=228528 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279493808 ingress_bytes=719667200 link_bytes=719667200 token_bytes=558759088 weight_movement_bytes=440401920 total_nvlink_bytes=999161008 weight_to_token_ratio=0.788179 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251619626e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=229248 combine_payload_bytes=279752704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279981952 ingress_bytes=389853184 link_bytes=389853184 token_bytes=559734656 weight_movement_bytes=110100480 total_nvlink_bytes=669835136 weight_to_token_ratio=0.196701 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780188068e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=230336 combine_payload_bytes=279537664 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279768000 ingress_bytes=499738624 link_bytes=499738624 token_bytes=559305664 weight_movement_bytes=220200960 total_nvlink_bytes=779506624 weight_to_token_ratio=0.393704 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.691276600e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277344256 dispatch_meta_bytes=229008 combine_payload_bytes=277344256 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277573264 ingress_bytes=497545216 link_bytes=497545216 token_bytes=554917520 weight_movement_bytes=220200960 total_nvlink_bytes=775118480 weight_to_token_ratio=0.396817 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.653129628e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44238 payload_rows_remote=38740 dup_rows=21298 dup_rows_remote=18656 dedup_ratio=0.324982 router_tokens_per_expert=[554, 512, 505, 517, 507, 504, 553, 493, 557, 513, 512, 500, 509, 512, 509, 500, 488, 475, 450, 507, 511, 514, 492, 564, 491, 513, 514, 507, 495, 565, 478, 507, 481, 500, 516, 475, 535, 525, 514, 544, 527, 496, 511, 513, 516, 526, 534, 496, 486, 493, 510, 540, 514, 536, 501, 531, 505, 516, 541, 526, 545, 472, 504, 511] payload_rows_per_dst=[5572, 5528, 5426, 5498, 5524, 5560, 5554, 5576] dup_rows_per_dst=[2718, 2696, 2576, 2642, 2656, 2678, 2668, 2664] combine_rows_per_src=[5572, 5528, 5426, 5498, 5524, 5560, 5554, 5576] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38754 dup_rows=21126 dup_rows_remote=18490 dedup_ratio=0.322357 router_tokens_per_expert=[539, 536, 490, 496, 531, 509, 513, 531, 482, 474, 515, 509, 520, 549, 516, 523, 507, 470, 504, 517, 504, 479, 522, 542, 505, 543, 485, 533, 478, 565, 527, 493, 500, 503, 495, 472, 513, 502, 539, 495, 492, 471, 531, 492, 548, 550, 517, 533, 487, 557, 519, 523, 503, 525, 519, 513, 490, 500, 484, 532, 526, 541, 491, 498] payload_rows_per_dst=[5638, 5556, 5510, 5530, 5432, 5548, 5656, 5540] dup_rows_per_dst=[2652, 2620, 2580, 2728, 2606, 2720, 2636, 2584] combine_rows_per_src=[5638, 5556, 5510, 5530, 5432, 5548, 5656, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44480 payload_rows_remote=38942 dup_rows=21056 dup_rows_remote=18374 dedup_ratio=0.321289 router_tokens_per_expert=[557, 489, 506, 475, 525, 502, 482, 531, 484, 506, 493, 518, 507, 493, 513, 491, 471, 521, 542, 520, 543, 512, 492, 509, 533, 548, 512, 530, 515, 487, 471, 507, 534, 512, 533, 489, 482, 523, 534, 496, 510, 486, 503, 543, 501, 574, 538, 526, 519, 490, 492, 518, 546, 517, 511, 555, 537, 475, 473, 489, 520, 524, 498, 535] payload_rows_per_dst=[5598, 5452, 5538, 5578, 5594, 5644, 5568, 5508] dup_rows_per_dst=[2536, 2558, 2682, 2628, 2612, 2718, 2728, 2594] combine_rows_per_src=[5598, 5452, 5538, 5578, 5594, 5644, 5568, 5508] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278104064 dispatch_meta_bytes=228568 combine_payload_bytes=278104064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278332632 ingress_bytes=388204544 link_bytes=388204544 token_bytes=556436696 weight_movement_bytes=110100480 total_nvlink_bytes=666537176 weight_to_token_ratio=0.197867 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751515507e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44288 payload_rows_remote=38782 dup_rows=21248 dup_rows_remote=18546 dedup_ratio=0.324219 router_tokens_per_expert=[532, 498, 535, 473, 511, 535, 492, 525, 532, 526, 518, 545, 476, 476, 493, 527, 501, 459, 491, 509, 555, 507, 473, 509, 496, 509, 496, 513, 508, 508, 514, 507, 510, 536, 556, 513, 480, 509, 514, 527, 513, 535, 487, 542, 539, 500, 492, 496, 487, 522, 495, 539, 515, 546, 527, 508, 523, 526, 468, 548, 500, 521, 515, 530] payload_rows_per_dst=[5554, 5526, 5452, 5504, 5564, 5506, 5592, 5590] dup_rows_per_dst=[2648, 2660, 2556, 2598, 2726, 2702, 2686, 2672] combine_rows_per_src=[5554, 5526, 5452, 5504, 5564, 5506, 5592, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278276096 dispatch_meta_bytes=228480 combine_payload_bytes=278276096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278504576 ingress_bytes=388376576 link_bytes=388376576 token_bytes=556780672 weight_movement_bytes=110100480 total_nvlink_bytes=666881152 weight_to_token_ratio=0.197745 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754507426e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280326144 dispatch_meta_bytes=229616 combine_payload_bytes=280326144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280555760 ingress_bytes=390426624 link_bytes=390426624 token_bytes=560881904 weight_movement_bytes=110100480 total_nvlink_bytes=670982384 weight_to_token_ratio=0.196299 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790161133e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38754 dup_rows=21126 dup_rows_remote=18490 dedup_ratio=0.322357 router_tokens_per_expert=[539, 536, 490, 496, 531, 509, 513, 531, 482, 474, 515, 509, 520, 549, 516, 523, 507, 470, 504, 517, 504, 479, 522, 542, 505, 543, 485, 533, 478, 565, 527, 493, 500, 503, 495, 472, 513, 502, 539, 495, 492, 471, 531, 492, 548, 550, 517, 533, 487, 557, 519, 523, 503, 525, 519, 513, 490, 500, 484, 532, 526, 541, 491, 498] payload_rows_per_dst=[5638, 5556, 5510, 5530, 5432, 5548, 5656, 5540] dup_rows_per_dst=[2652, 2620, 2580, 2728, 2606, 2720, 2636, 2584] combine_rows_per_src=[5638, 5556, 5510, 5530, 5432, 5548, 5656, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44238 payload_rows_remote=38740 dup_rows=21298 dup_rows_remote=18656 dedup_ratio=0.324982 router_tokens_per_expert=[554, 512, 505, 517, 507, 504, 553, 493, 557, 513, 512, 500, 509, 512, 509, 500, 488, 475, 450, 507, 511, 514, 492, 564, 491, 513, 514, 507, 495, 565, 478, 507, 481, 500, 516, 475, 535, 525, 514, 544, 527, 496, 511, 513, 516, 526, 534, 496, 486, 493, 510, 540, 514, 536, 501, 531, 505, 516, 541, 526, 545, 472, 504, 511] payload_rows_per_dst=[5572, 5528, 5426, 5498, 5524, 5560, 5554, 5576] dup_rows_per_dst=[2718, 2696, 2576, 2642, 2656, 2678, 2668, 2664] combine_rows_per_src=[5572, 5528, 5426, 5498, 5524, 5560, 5554, 5576] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277688320 dispatch_meta_bytes=229584 combine_payload_bytes=277688320 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277917904 ingress_bytes=497889280 link_bytes=497889280 token_bytes=555606224 weight_movement_bytes=220200960 total_nvlink_bytes=775807184 weight_to_token_ratio=0.396326 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.659113466e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277788672 dispatch_meta_bytes=228976 combine_payload_bytes=277788672 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278017648 ingress_bytes=608090112 link_bytes=608090112 token_bytes=555806320 weight_movement_bytes=330301440 total_nvlink_bytes=886107760 weight_to_token_ratio=0.594274 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057568718e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=229264 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279365520 ingress_bytes=389236736 link_bytes=389236736 token_bytes=558501776 weight_movement_bytes=110100480 total_nvlink_bytes=668602256 weight_to_token_ratio=0.197135 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769467023e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277989376 dispatch_meta_bytes=229312 combine_payload_bytes=277989376 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278218688 ingress_bytes=498190336 link_bytes=498190336 token_bytes=556208064 weight_movement_bytes=220200960 total_nvlink_bytes=776409024 weight_to_token_ratio=0.395897 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.664349325e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44714 payload_rows_remote=39108 dup_rows=20822 dup_rows_remote=18296 dedup_ratio=0.317719 router_tokens_per_expert=[488, 512, 518, 562, 540, 518, 504, 523, 539, 515, 488, 490, 492, 512, 503, 527, 546, 491, 503, 500, 526, 509, 533, 546, 535, 526, 552, 531, 494, 565, 527, 475, 529, 499, 528, 466, 527, 511, 493, 508, 539, 485, 504, 484, 478, 499, 512, 532, 476, 488, 493, 499, 527, 527, 519, 519, 479, 503, 492, 488, 513, 502, 509, 550] payload_rows_per_dst=[5654, 5606, 5652, 5642, 5498, 5558, 5538, 5566] dup_rows_per_dst=[2676, 2526, 2656, 2768, 2624, 2508, 2558, 2506] combine_rows_per_src=[5654, 5606, 5652, 5642, 5498, 5558, 5538, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277788672 dispatch_meta_bytes=228976 combine_payload_bytes=277788672 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278017648 ingress_bytes=608090112 link_bytes=608090112 token_bytes=555806320 weight_movement_bytes=330301440 total_nvlink_bytes=886107760 weight_to_token_ratio=0.594274 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057568718e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=38798 dup_rows=21110 dup_rows_remote=18344 dedup_ratio=0.322113 router_tokens_per_expert=[506, 482, 531, 498, 545, 519, 487, 504, 489, 500, 545, 491, 500, 533, 513, 510, 521, 521, 509, 505, 531, 491, 543, 531, 511, 497, 511, 518, 512, 493, 481, 505, 531, 507, 547, 537, 523, 539, 520, 493, 502, 485, 511, 532, 486, 518, 520, 489, 504, 536, 516, 524, 533, 548, 510, 514, 491, 557, 495, 516, 474, 464, 527, 486] payload_rows_per_dst=[5532, 5484, 5592, 5478, 5628, 5602, 5658, 5452] dup_rows_per_dst=[2612, 2678, 2712, 2578, 2766, 2484, 2712, 2568] combine_rows_per_src=[5532, 5484, 5592, 5478, 5628, 5602, 5658, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277688320 dispatch_meta_bytes=229584 combine_payload_bytes=277688320 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277917904 ingress_bytes=497889280 link_bytes=497889280 token_bytes=555606224 weight_movement_bytes=220200960 total_nvlink_bytes=775807184 weight_to_token_ratio=0.396326 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.659113466e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44480 payload_rows_remote=38942 dup_rows=21056 dup_rows_remote=18374 dedup_ratio=0.321289 router_tokens_per_expert=[557, 489, 506, 475, 525, 502, 482, 531, 484, 506, 493, 518, 507, 493, 513, 491, 471, 521, 542, 520, 543, 512, 492, 509, 533, 548, 512, 530, 515, 487, 471, 507, 534, 512, 533, 489, 482, 523, 534, 496, 510, 486, 503, 543, 501, 574, 538, 526, 519, 490, 492, 518, 546, 517, 511, 555, 537, 475, 473, 489, 520, 524, 498, 535] payload_rows_per_dst=[5598, 5452, 5538, 5578, 5594, 5644, 5568, 5508] dup_rows_per_dst=[2536, 2558, 2682, 2628, 2612, 2718, 2728, 2594] combine_rows_per_src=[5598, 5452, 5538, 5578, 5594, 5644, 5568, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38822 dup_rows=21094 dup_rows_remote=18298 dedup_ratio=0.321869 router_tokens_per_expert=[504, 520, 504, 512, 485, 493, 514, 542, 497, 536, 509, 511, 561, 489, 503, 493, 526, 473, 534, 506, 536, 489, 522, 487, 486, 514, 514, 545, 522, 508, 527, 538, 485, 486, 534, 483, 513, 486, 531, 519, 494, 488, 502, 476, 544, 544, 530, 522, 524, 499, 522, 497, 490, 526, 527, 509, 528, 534, 517, 542, 523, 482, 492, 519] payload_rows_per_dst=[5532, 5568, 5542, 5554, 5502, 5528, 5596, 5620] dup_rows_per_dst=[2502, 2736, 2604, 2612, 2572, 2678, 2594, 2796] combine_rows_per_src=[5532, 5568, 5542, 5554, 5502, 5528, 5596, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=39010 dup_rows=21060 dup_rows_remote=18414 dedup_ratio=0.321350 router_tokens_per_expert=[529, 518, 483, 470, 528, 505, 523, 500, 546, 514, 536, 495, 525, 523, 513, 519, 485, 507, 466, 516, 515, 545, 498, 557, 523, 492, 488, 526, 512, 513, 523, 509, 543, 497, 490, 553, 471, 518, 516, 513, 517, 483, 508, 514, 522, 506, 494, 522, 510, 501, 488, 489, 514, 547, 475, 525, 503, 547, 538, 480, 499, 533, 532, 518] payload_rows_per_dst=[5466, 5658, 5690, 5596, 5582, 5494, 5384, 5606] dup_rows_per_dst=[2646, 2684, 2768, 2602, 2650, 2608, 2408, 2694] combine_rows_per_src=[5466, 5658, 5690, 5596, 5582, 5494, 5384, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280326144 dispatch_meta_bytes=229616 combine_payload_bytes=280326144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280555760 ingress_bytes=390426624 link_bytes=390426624 token_bytes=560881904 weight_movement_bytes=110100480 total_nvlink_bytes=670982384 weight_to_token_ratio=0.196299 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790161133e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=39082 dup_rows=20984 dup_rows_remote=18440 dedup_ratio=0.320190 router_tokens_per_expert=[518, 501, 492, 530, 546, 489, 512, 511, 517, 548, 508, 525, 526, 518, 535, 534, 488, 517, 558, 478, 521, 506, 497, 524, 492, 477, 491, 516, 561, 470, 504, 505, 495, 545, 510, 499, 512, 484, 476, 486, 514, 534, 506, 509, 534, 502, 520, 532, 526, 513, 510, 469, 507, 493, 515, 521, 492, 536, 491, 531, 518, 521, 519, 533] payload_rows_per_dst=[5560, 5694, 5590, 5494, 5470, 5674, 5510, 5560] dup_rows_per_dst=[2638, 2728, 2588, 2538, 2544, 2628, 2598, 2722] combine_rows_per_src=[5560, 5694, 5590, 5494, 5470, 5674, 5510, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44604 payload_rows_remote=38918 dup_rows=20932 dup_rows_remote=18248 dedup_ratio=0.319397 router_tokens_per_expert=[509, 499, 509, 496, 520, 507, 519, 489, 483, 503, 502, 477, 521, 550, 523, 543, 535, 503, 535, 465, 490, 528, 483, 505, 518, 521, 514, 513, 536, 487, 518, 515, 523, 526, 532, 482, 506, 492, 499, 524, 516, 537, 527, 551, 513, 506, 514, 522, 464, 550, 491, 510, 470, 533, 497, 504, 524, 513, 533, 497, 549, 506, 529, 512] payload_rows_per_dst=[5486, 5606, 5520, 5620, 5518, 5616, 5552, 5686] dup_rows_per_dst=[2558, 2598, 2480, 2624, 2650, 2764, 2574, 2684] combine_rows_per_src=[5486, 5606, 5520, 5620, 5518, 5616, 5552, 5686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278104064 dispatch_meta_bytes=228568 combine_payload_bytes=278104064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278332632 ingress_bytes=388204544 link_bytes=388204544 token_bytes=556436696 weight_movement_bytes=110100480 total_nvlink_bytes=666537176 weight_to_token_ratio=0.197867 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751515507e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=229264 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279365520 ingress_bytes=389236736 link_bytes=389236736 token_bytes=558501776 weight_movement_bytes=110100480 total_nvlink_bytes=668602256 weight_to_token_ratio=0.197135 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769467023e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278276096 dispatch_meta_bytes=228480 combine_payload_bytes=278276096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278504576 ingress_bytes=388376576 link_bytes=388376576 token_bytes=556780672 weight_movement_bytes=110100480 total_nvlink_bytes=666881152 weight_to_token_ratio=0.197745 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754507426e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=229696 combine_payload_bytes=279623680 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279853376 ingress_bytes=609925120 link_bytes=609925120 token_bytes=559477056 weight_movement_bytes=330301440 total_nvlink_bytes=889778496 weight_to_token_ratio=0.590375 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060760099e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38986 dup_rows=21086 dup_rows_remote=18502 dedup_ratio=0.321747 router_tokens_per_expert=[514, 535, 522, 515, 497, 506, 508, 500, 503, 497, 483, 500, 518, 495, 551, 477, 487, 501, 507, 543, 504, 497, 512, 475, 459, 519, 507, 493, 490, 528, 551, 484, 529, 509, 510, 522, 487, 490, 536, 515, 544, 535, 509, 526, 515, 508, 561, 506, 539, 511, 500, 507, 532, 533, 515, 538, 491, 550, 514, 503, 524, 499, 541, 491] payload_rows_per_dst=[5564, 5464, 5546, 5478, 5598, 5660, 5548, 5592] dup_rows_per_dst=[2630, 2584, 2506, 2584, 2598, 2748, 2802, 2634] combine_rows_per_src=[5564, 5464, 5546, 5478, 5598, 5660, 5548, 5592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=39034 dup_rows=20926 dup_rows_remote=18442 dedup_ratio=0.319305 router_tokens_per_expert=[536, 522, 527, 495, 522, 543, 516, 529, 484, 531, 499, 508, 529, 519, 522, 498, 503, 505, 530, 527, 555, 527, 482, 507, 535, 486, 530, 525, 489, 462, 480, 523, 535, 530, 476, 497, 511, 474, 534, 508, 531, 474, 488, 521, 511, 469, 513, 528, 548, 486, 484, 521, 501, 518, 519, 494, 509, 527, 541, 528, 497, 517, 513, 519] payload_rows_per_dst=[5620, 5544, 5606, 5576, 5570, 5530, 5564, 5600] dup_rows_per_dst=[2760, 2636, 2666, 2484, 2560, 2540, 2578, 2702] combine_rows_per_src=[5620, 5544, 5606, 5576, 5570, 5530, 5564, 5600] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44656 payload_rows_remote=39146 dup_rows=20880 dup_rows_remote=18304 dedup_ratio=0.318604 router_tokens_per_expert=[476, 504, 500, 500, 512, 518, 490, 501, 509, 512, 531, 486, 553, 476, 509, 497, 519, 512, 535, 546, 505, 518, 500, 512, 513, 518, 528, 573, 545, 463, 508, 547, 531, 481, 528, 506, 496, 508, 514, 545, 498, 503, 514, 515, 485, 494, 514, 520, 525, 513, 487, 490, 533, 466, 533, 496, 555, 500, 497, 550, 510, 510, 500, 535] payload_rows_per_dst=[5500, 5546, 5636, 5644, 5620, 5566, 5510, 5634] dup_rows_per_dst=[2502, 2600, 2658, 2746, 2598, 2520, 2576, 2680] combine_rows_per_src=[5500, 5546, 5636, 5644, 5620, 5566, 5510, 5634] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38954 dup_rows=20978 dup_rows_remote=18384 dedup_ratio=0.320099 router_tokens_per_expert=[520, 504, 485, 528, 485, 515, 533, 509, 481, 523, 478, 518, 535, 482, 521, 478, 480, 547, 509, 505, 500, 508, 540, 510, 489, 471, 511, 549, 513, 517, 513, 549, 503, 521, 499, 483, 509, 516, 537, 491, 508, 512, 532, 478, 537, 525, 511, 520, 531, 523, 521, 535, 532, 501, 540, 497, 518, 490, 512, 527, 535, 494, 523, 501] payload_rows_per_dst=[5538, 5504, 5604, 5594, 5552, 5618, 5584, 5564] dup_rows_per_dst=[2620, 2528, 2594, 2630, 2566, 2628, 2776, 2636] combine_rows_per_src=[5538, 5504, 5604, 5594, 5552, 5618, 5584, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280139776 dispatch_meta_bytes=230088 combine_payload_bytes=280139776 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280369864 ingress_bytes=390240256 link_bytes=390240256 token_bytes=560509640 weight_movement_bytes=110100480 total_nvlink_bytes=670610120 weight_to_token_ratio=0.196429 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786919887e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=38988 dup_rows=21060 dup_rows_remote=18412 dedup_ratio=0.321350 router_tokens_per_expert=[476, 541, 490, 530, 523, 529, 506, 554, 552, 503, 524, 496, 497, 487, 497, 530, 496, 515, 536, 496, 556, 527, 467, 518, 500, 483, 519, 519, 514, 517, 511, 501, 505, 524, 540, 500, 504, 551, 528, 526, 523, 485, 529, 544, 532, 492, 466, 497, 479, 536, 513, 499, 500, 528, 542, 549, 492, 494, 479, 507, 459, 510, 492, 533] payload_rows_per_dst=[5526, 5568, 5598, 5534, 5670, 5488, 5648, 5444] dup_rows_per_dst=[2772, 2604, 2624, 2594, 2686, 2648, 2644, 2488] combine_rows_per_src=[5526, 5568, 5598, 5534, 5670, 5488, 5648, 5444] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=228664 combine_payload_bytes=278964224 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279192888 ingress_bytes=499165184 link_bytes=499165184 token_bytes=558157112 weight_movement_bytes=220200960 total_nvlink_bytes=778358072 weight_to_token_ratio=0.394514 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681303535e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279451648 dispatch_meta_bytes=229952 combine_payload_bytes=279451648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279681600 ingress_bytes=389552128 link_bytes=389552128 token_bytes=559133248 weight_movement_bytes=110100480 total_nvlink_bytes=669233728 weight_to_token_ratio=0.196913 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774952209e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280598528 dispatch_meta_bytes=229800 combine_payload_bytes=280598528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280828328 ingress_bytes=500799488 link_bytes=500799488 token_bytes=561426856 weight_movement_bytes=220200960 total_nvlink_bytes=781627816 weight_to_token_ratio=0.392217 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.709726770e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229904 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280025616 ingress_bytes=389896192 link_bytes=389896192 token_bytes=559821328 weight_movement_bytes=110100480 total_nvlink_bytes=669921808 weight_to_token_ratio=0.196671 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780936048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229352 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279451624 ingress_bytes=499423232 link_bytes=499423232 token_bytes=558673896 weight_movement_bytes=220200960 total_nvlink_bytes=778874856 weight_to_token_ratio=0.394149 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685791415e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280598528 dispatch_meta_bytes=229800 combine_payload_bytes=280598528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280828328 ingress_bytes=500799488 link_bytes=500799488 token_bytes=561426856 weight_movement_bytes=220200960 total_nvlink_bytes=781627816 weight_to_token_ratio=0.392217 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.709726770e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38986 dup_rows=21086 dup_rows_remote=18502 dedup_ratio=0.321747 router_tokens_per_expert=[514, 535, 522, 515, 497, 506, 508, 500, 503, 497, 483, 500, 518, 495, 551, 477, 487, 501, 507, 543, 504, 497, 512, 475, 459, 519, 507, 493, 490, 528, 551, 484, 529, 509, 510, 522, 487, 490, 536, 515, 544, 535, 509, 526, 515, 508, 561, 506, 539, 511, 500, 507, 532, 533, 515, 538, 491, 550, 514, 503, 524, 499, 541, 491] payload_rows_per_dst=[5564, 5464, 5546, 5478, 5598, 5660, 5548, 5592] dup_rows_per_dst=[2630, 2584, 2506, 2584, 2598, 2748, 2802, 2634] combine_rows_per_src=[5564, 5464, 5546, 5478, 5598, 5660, 5548, 5592] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=229600 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279695584 ingress_bytes=609767424 link_bytes=609767424 token_bytes=559161568 weight_movement_bytes=330301440 total_nvlink_bytes=889463008 weight_to_token_ratio=0.590708 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060485840e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=39082 dup_rows=20984 dup_rows_remote=18440 dedup_ratio=0.320190 router_tokens_per_expert=[518, 501, 492, 530, 546, 489, 512, 511, 517, 548, 508, 525, 526, 518, 535, 534, 488, 517, 558, 478, 521, 506, 497, 524, 492, 477, 491, 516, 561, 470, 504, 505, 495, 545, 510, 499, 512, 484, 476, 486, 514, 534, 506, 509, 534, 502, 520, 532, 526, 513, 510, 469, 507, 493, 515, 521, 492, 536, 491, 531, 518, 521, 519, 533] payload_rows_per_dst=[5560, 5694, 5590, 5494, 5470, 5674, 5510, 5560] dup_rows_per_dst=[2638, 2728, 2588, 2538, 2544, 2628, 2598, 2722] combine_rows_per_src=[5560, 5694, 5590, 5494, 5470, 5674, 5510, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44604 payload_rows_remote=38918 dup_rows=20932 dup_rows_remote=18248 dedup_ratio=0.319397 router_tokens_per_expert=[509, 499, 509, 496, 520, 507, 519, 489, 483, 503, 502, 477, 521, 550, 523, 543, 535, 503, 535, 465, 490, 528, 483, 505, 518, 521, 514, 513, 536, 487, 518, 515, 523, 526, 532, 482, 506, 492, 499, 524, 516, 537, 527, 551, 513, 506, 514, 522, 464, 550, 491, 510, 470, 533, 497, 504, 524, 513, 533, 497, 549, 506, 529, 512] payload_rows_per_dst=[5486, 5606, 5520, 5620, 5518, 5616, 5552, 5686] dup_rows_per_dst=[2558, 2598, 2480, 2624, 2650, 2764, 2574, 2684] combine_rows_per_src=[5486, 5606, 5520, 5620, 5518, 5616, 5552, 5686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38954 dup_rows=20978 dup_rows_remote=18384 dedup_ratio=0.320099 router_tokens_per_expert=[520, 504, 485, 528, 485, 515, 533, 509, 481, 523, 478, 518, 535, 482, 521, 478, 480, 547, 509, 505, 500, 508, 540, 510, 489, 471, 511, 549, 513, 517, 513, 549, 503, 521, 499, 483, 509, 516, 537, 491, 508, 512, 532, 478, 537, 525, 511, 520, 531, 523, 521, 535, 532, 501, 540, 497, 518, 490, 512, 527, 535, 494, 523, 501] payload_rows_per_dst=[5538, 5504, 5604, 5594, 5552, 5618, 5584, 5564] dup_rows_per_dst=[2620, 2528, 2594, 2630, 2566, 2628, 2776, 2636] combine_rows_per_src=[5538, 5504, 5604, 5594, 5552, 5618, 5584, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38828 dup_rows=21090 dup_rows_remote=18446 dedup_ratio=0.321808 router_tokens_per_expert=[520, 524, 522, 512, 458, 556, 522, 517, 502, 498, 518, 503, 537, 508, 500, 513, 512, 518, 512, 537, 511, 491, 551, 512, 538, 526, 536, 471, 520, 485, 522, 484, 503, 514, 525, 466, 519, 515, 494, 487, 536, 469, 531, 518, 493, 511, 496, 561, 500, 545, 493, 534, 508, 533, 507, 532, 473, 505, 509, 495, 534, 496, 515, 515] payload_rows_per_dst=[5618, 5622, 5556, 5622, 5626, 5448, 5470, 5484] dup_rows_per_dst=[2644, 2710, 2764, 2774, 2710, 2460, 2602, 2426] combine_rows_per_src=[5618, 5622, 5556, 5622, 5626, 5448, 5470, 5484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38788 dup_rows=21068 dup_rows_remote=18368 dedup_ratio=0.321472 router_tokens_per_expert=[539, 507, 518, 511, 490, 491, 506, 493, 511, 507, 490, 490, 543, 504, 514, 545, 494, 518, 493, 503, 449, 515, 514, 527, 554, 554, 478, 486, 509, 486, 486, 518, 491, 482, 532, 544, 552, 531, 509, 549, 511, 575, 525, 511, 493, 534, 521, 513, 503, 506, 475, 468, 538, 503, 537, 488, 522, 523, 520, 502, 562, 514, 491, 500] payload_rows_per_dst=[5530, 5544, 5510, 5534, 5680, 5608, 5492, 5570] dup_rows_per_dst=[2580, 2664, 2516, 2608, 2700, 2758, 2544, 2698] combine_rows_per_src=[5530, 5544, 5510, 5534, 5680, 5608, 5492, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279451648 dispatch_meta_bytes=229952 combine_payload_bytes=279451648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279681600 ingress_bytes=389552128 link_bytes=389552128 token_bytes=559133248 weight_movement_bytes=110100480 total_nvlink_bytes=669233728 weight_to_token_ratio=0.196913 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774952209e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=38882 dup_rows=20950 dup_rows_remote=18160 dedup_ratio=0.319672 router_tokens_per_expert=[468, 476, 509, 504, 490, 480, 555, 493, 502, 501, 522, 518, 478, 534, 499, 523, 502, 520, 480, 508, 501, 489, 497, 549, 536, 513, 494, 479, 513, 511, 532, 504, 512, 528, 545, 449, 533, 502, 483, 530, 501, 536, 519, 481, 507, 505, 537, 486, 543, 500, 546, 540, 530, 554, 507, 528, 529, 537, 537, 505, 537, 511, 536, 494] payload_rows_per_dst=[5488, 5422, 5554, 5624, 5428, 5660, 5706, 5704] dup_rows_per_dst=[2462, 2452, 2538, 2540, 2638, 2740, 2790, 2790] combine_rows_per_src=[5488, 5422, 5554, 5624, 5428, 5660, 5706, 5704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280139776 dispatch_meta_bytes=230088 combine_payload_bytes=280139776 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280369864 ingress_bytes=390240256 link_bytes=390240256 token_bytes=560509640 weight_movement_bytes=110100480 total_nvlink_bytes=670610120 weight_to_token_ratio=0.196429 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786919887e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229352 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279451624 ingress_bytes=499423232 link_bytes=499423232 token_bytes=558673896 weight_movement_bytes=220200960 total_nvlink_bytes=778874856 weight_to_token_ratio=0.394149 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685791415e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=228664 combine_payload_bytes=278964224 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279192888 ingress_bytes=499165184 link_bytes=499165184 token_bytes=558157112 weight_movement_bytes=220200960 total_nvlink_bytes=778358072 weight_to_token_ratio=0.394514 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681303535e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44660 payload_rows_remote=39062 dup_rows=20876 dup_rows_remote=18186 dedup_ratio=0.318542 router_tokens_per_expert=[526, 513, 516, 526, 503, 485, 501, 501, 531, 512, 503, 577, 502, 527, 502, 514, 524, 514, 523, 491, 521, 502, 527, 493, 470, 497, 510, 543, 529, 528, 519, 548, 499, 517, 544, 514, 525, 467, 494, 505, 526, 491, 478, 534, 471, 513, 473, 502, 516, 512, 502, 494, 525, 514, 517, 525, 555, 505, 505, 507, 516, 533, 510, 501] payload_rows_per_dst=[5510, 5550, 5648, 5598, 5568, 5476, 5636, 5674] dup_rows_per_dst=[2632, 2786, 2542, 2690, 2562, 2500, 2574, 2590] combine_rows_per_src=[5510, 5550, 5648, 5598, 5568, 5476, 5636, 5674] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278319104 dispatch_meta_bytes=229096 combine_payload_bytes=278319104 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278548200 ingress_bytes=498520064 link_bytes=498520064 token_bytes=556867304 weight_movement_bytes=220200960 total_nvlink_bytes=777068264 weight_to_token_ratio=0.395428 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670083838e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44576 payload_rows_remote=39030 dup_rows=20960 dup_rows_remote=18262 dedup_ratio=0.319824 router_tokens_per_expert=[514, 529, 516, 513, 538, 498, 514, 532, 518, 539, 505, 511, 527, 485, 515, 522, 541, 456, 489, 474, 499, 545, 487, 529, 489, 533, 523, 498, 517, 530, 512, 486, 531, 484, 517, 522, 528, 489, 526, 490, 545, 552, 521, 492, 516, 483, 519, 504, 503, 534, 475, 545, 521, 482, 514, 503, 511, 511, 512, 487, 494, 535, 528, 510] payload_rows_per_dst=[5660, 5546, 5518, 5602, 5532, 5608, 5484, 5626] dup_rows_per_dst=[2648, 2698, 2522, 2574, 2642, 2656, 2670, 2550] combine_rows_per_src=[5660, 5546, 5518, 5602, 5532, 5608, 5484, 5626] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44384 payload_rows_remote=38748 dup_rows=21152 dup_rows_remote=18454 dedup_ratio=0.322754 router_tokens_per_expert=[531, 484, 506, 514, 499, 514, 481, 517, 512, 514, 512, 522, 500, 492, 538, 511, 500, 522, 469, 468, 509, 504, 536, 526, 525, 532, 520, 513, 498, 540, 546, 483, 484, 509, 518, 468, 540, 510, 538, 480, 513, 500, 490, 511, 507, 498, 511, 548, 511, 557, 474, 534, 532, 498, 549, 512, 526, 502, 486, 523, 513, 526, 535, 527] payload_rows_per_dst=[5450, 5536, 5528, 5582, 5544, 5538, 5636, 5570] dup_rows_per_dst=[2642, 2666, 2540, 2732, 2550, 2618, 2698, 2706] combine_rows_per_src=[5450, 5536, 5528, 5582, 5544, 5538, 5636, 5570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44660 payload_rows_remote=39062 dup_rows=20876 dup_rows_remote=18186 dedup_ratio=0.318542 router_tokens_per_expert=[526, 513, 516, 526, 503, 485, 501, 501, 531, 512, 503, 577, 502, 527, 502, 514, 524, 514, 523, 491, 521, 502, 527, 493, 470, 497, 510, 543, 529, 528, 519, 548, 499, 517, 544, 514, 525, 467, 494, 505, 526, 491, 478, 534, 471, 513, 473, 502, 516, 512, 502, 494, 525, 514, 517, 525, 555, 505, 505, 507, 516, 533, 510, 501] payload_rows_per_dst=[5510, 5550, 5648, 5598, 5568, 5476, 5636, 5674] dup_rows_per_dst=[2632, 2786, 2542, 2690, 2562, 2500, 2574, 2590] combine_rows_per_src=[5510, 5550, 5648, 5598, 5568, 5476, 5636, 5674] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278032384 dispatch_meta_bytes=228624 combine_payload_bytes=278032384 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278261008 ingress_bytes=498233344 link_bytes=498233344 token_bytes=556293392 weight_movement_bytes=220200960 total_nvlink_bytes=776494352 weight_to_token_ratio=0.395836 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.665097305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44488 payload_rows_remote=38952 dup_rows=21048 dup_rows_remote=18394 dedup_ratio=0.321167 router_tokens_per_expert=[497, 506, 504, 504, 490, 510, 512, 475, 476, 552, 507, 572, 525, 529, 511, 510, 526, 521, 532, 510, 500, 478, 526, 502, 516, 530, 507, 477, 529, 515, 509, 477, 546, 520, 476, 505, 525, 497, 523, 524, 511, 522, 547, 508, 513, 506, 522, 511, 532, 507, 526, 481, 511, 579, 498, 463, 531, 474, 475, 487, 530, 544, 537, 502] payload_rows_per_dst=[5512, 5650, 5536, 5562, 5612, 5546, 5548, 5522] dup_rows_per_dst=[2484, 2714, 2654, 2558, 2620, 2734, 2646, 2638] combine_rows_per_src=[5512, 5650, 5536, 5562, 5612, 5546, 5548, 5522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=228168 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278934344 ingress_bytes=498907136 link_bytes=498907136 token_bytes=557640520 weight_movement_bytes=220200960 total_nvlink_bytes=777841480 weight_to_token_ratio=0.394880 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676815656e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44318 payload_rows_remote=38758 dup_rows=21218 dup_rows_remote=18664 dedup_ratio=0.323761 router_tokens_per_expert=[509, 479, 500, 513, 487, 526, 558, 507, 480, 496, 469, 525, 520, 514, 501, 505, 503, 545, 513, 491, 526, 517, 527, 498, 491, 522, 528, 509, 514, 540, 540, 520, 519, 495, 516, 522, 490, 518, 575, 562, 471, 531, 506, 500, 510, 521, 523, 495, 505, 539, 500, 506, 511, 504, 514, 495, 533, 509, 492, 499, 495, 514, 491, 534] payload_rows_per_dst=[5534, 5430, 5614, 5526, 5570, 5560, 5514, 5570] dup_rows_per_dst=[2624, 2590, 2626, 2802, 2824, 2554, 2634, 2564] combine_rows_per_src=[5534, 5430, 5614, 5526, 5570, 5560, 5514, 5570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=229168 combine_payload_bytes=279767040 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279996208 ingress_bytes=610068480 link_bytes=610068480 token_bytes=559763248 weight_movement_bytes=330301440 total_nvlink_bytes=890064688 weight_to_token_ratio=0.590073 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061009426e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277745664 dispatch_meta_bytes=228808 combine_payload_bytes=277745664 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=277974472 ingress_bytes=387846144 link_bytes=387846144 token_bytes=555720136 weight_movement_bytes=110100480 total_nvlink_bytes=665820616 weight_to_token_ratio=0.198122 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745282341e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279996416 dispatch_meta_bytes=228992 combine_payload_bytes=279996416 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280225408 ingress_bytes=610297856 link_bytes=610297856 token_bytes=560221824 weight_movement_bytes=330301440 total_nvlink_bytes=890523264 weight_to_token_ratio=0.589590 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061408348e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229384 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279437320 ingress_bytes=389308416 link_bytes=389308416 token_bytes=558645256 weight_movement_bytes=110100480 total_nvlink_bytes=668745736 weight_to_token_ratio=0.197085 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770713657e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277745664 dispatch_meta_bytes=228808 combine_payload_bytes=277745664 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=277974472 ingress_bytes=387846144 link_bytes=387846144 token_bytes=555720136 weight_movement_bytes=110100480 total_nvlink_bytes=665820616 weight_to_token_ratio=0.198122 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745282341e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44576 payload_rows_remote=39030 dup_rows=20960 dup_rows_remote=18262 dedup_ratio=0.319824 router_tokens_per_expert=[514, 529, 516, 513, 538, 498, 514, 532, 518, 539, 505, 511, 527, 485, 515, 522, 541, 456, 489, 474, 499, 545, 487, 529, 489, 533, 523, 498, 517, 530, 512, 486, 531, 484, 517, 522, 528, 489, 526, 490, 545, 552, 521, 492, 516, 483, 519, 504, 503, 534, 475, 545, 521, 482, 514, 503, 511, 511, 512, 487, 494, 535, 528, 510] payload_rows_per_dst=[5660, 5546, 5518, 5602, 5532, 5608, 5484, 5626] dup_rows_per_dst=[2648, 2698, 2522, 2574, 2642, 2656, 2670, 2550] combine_rows_per_src=[5660, 5546, 5518, 5602, 5532, 5608, 5484, 5626] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277817344 dispatch_meta_bytes=229688 combine_payload_bytes=277817344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278047032 ingress_bytes=387917824 link_bytes=387917824 token_bytes=555864376 weight_movement_bytes=110100480 total_nvlink_bytes=665964856 weight_to_token_ratio=0.198071 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746528975e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38788 dup_rows=21068 dup_rows_remote=18368 dedup_ratio=0.321472 router_tokens_per_expert=[539, 507, 518, 511, 490, 491, 506, 493, 511, 507, 490, 490, 543, 504, 514, 545, 494, 518, 493, 503, 449, 515, 514, 527, 554, 554, 478, 486, 509, 486, 486, 518, 491, 482, 532, 544, 552, 531, 509, 549, 511, 575, 525, 511, 493, 534, 521, 513, 503, 506, 475, 468, 538, 503, 537, 488, 522, 523, 520, 502, 562, 514, 491, 500] payload_rows_per_dst=[5530, 5544, 5510, 5534, 5680, 5608, 5492, 5570] dup_rows_per_dst=[2580, 2664, 2516, 2608, 2700, 2758, 2544, 2698] combine_rows_per_src=[5530, 5544, 5510, 5534, 5680, 5608, 5492, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=38882 dup_rows=20950 dup_rows_remote=18160 dedup_ratio=0.319672 router_tokens_per_expert=[468, 476, 509, 504, 490, 480, 555, 493, 502, 501, 522, 518, 478, 534, 499, 523, 502, 520, 480, 508, 501, 489, 497, 549, 536, 513, 494, 479, 513, 511, 532, 504, 512, 528, 545, 449, 533, 502, 483, 530, 501, 536, 519, 481, 507, 505, 537, 486, 543, 500, 546, 540, 530, 554, 507, 528, 529, 537, 537, 505, 537, 511, 536, 494] payload_rows_per_dst=[5488, 5422, 5554, 5624, 5428, 5660, 5706, 5704] dup_rows_per_dst=[2462, 2452, 2538, 2540, 2638, 2740, 2790, 2790] combine_rows_per_src=[5488, 5422, 5554, 5624, 5428, 5660, 5706, 5704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44488 payload_rows_remote=38952 dup_rows=21048 dup_rows_remote=18394 dedup_ratio=0.321167 router_tokens_per_expert=[497, 506, 504, 504, 490, 510, 512, 475, 476, 552, 507, 572, 525, 529, 511, 510, 526, 521, 532, 510, 500, 478, 526, 502, 516, 530, 507, 477, 529, 515, 509, 477, 546, 520, 476, 505, 525, 497, 523, 524, 511, 522, 547, 508, 513, 506, 522, 511, 532, 507, 526, 481, 511, 579, 498, 463, 531, 474, 475, 487, 530, 544, 537, 502] payload_rows_per_dst=[5512, 5650, 5536, 5562, 5612, 5546, 5548, 5522] dup_rows_per_dst=[2484, 2714, 2654, 2558, 2620, 2734, 2646, 2638] combine_rows_per_src=[5512, 5650, 5536, 5562, 5612, 5546, 5548, 5522] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=38808 dup_rows=20936 dup_rows_remote=17948 dedup_ratio=0.319458 router_tokens_per_expert=[491, 518, 540, 526, 530, 524, 521, 500, 507, 500, 528, 536, 494, 558, 528, 489, 531, 556, 535, 472, 530, 492, 529, 538, 504, 539, 524, 515, 530, 538, 529, 488, 503, 444, 493, 527, 525, 489, 547, 505, 535, 470, 503, 485, 540, 476, 466, 515, 474, 535, 507, 472, 547, 481, 496, 471, 505, 537, 477, 490, 517, 525, 538, 533] payload_rows_per_dst=[5792, 5618, 5802, 5526, 5458, 5518, 5472, 5414] dup_rows_per_dst=[2988, 2670, 2888, 2478, 2608, 2462, 2492, 2350] combine_rows_per_src=[5792, 5618, 5802, 5526, 5458, 5518, 5472, 5414] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44346 payload_rows_remote=38974 dup_rows=21190 dup_rows_remote=18620 dedup_ratio=0.323334 router_tokens_per_expert=[498, 539, 504, 492, 516, 479, 530, 526, 508, 500, 512, 499, 512, 486, 506, 494, 546, 511, 518, 542, 516, 525, 504, 480, 510, 513, 515, 502, 507, 519, 519, 500, 513, 526, 494, 461, 482, 514, 525, 456, 531, 521, 512, 558, 564, 477, 524, 532, 498, 527, 517, 534, 533, 530, 462, 498, 481, 511, 536, 523, 532, 547, 505, 516] payload_rows_per_dst=[5500, 5550, 5568, 5508, 5372, 5676, 5524, 5648] dup_rows_per_dst=[2668, 2484, 2716, 2662, 2570, 2762, 2674, 2654] combine_rows_per_src=[5500, 5550, 5568, 5508, 5372, 5676, 5524, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=229168 combine_payload_bytes=279767040 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279996208 ingress_bytes=610068480 link_bytes=610068480 token_bytes=559763248 weight_movement_bytes=330301440 total_nvlink_bytes=890064688 weight_to_token_ratio=0.590073 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061009426e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38924 dup_rows=20984 dup_rows_remote=18316 dedup_ratio=0.320190 router_tokens_per_expert=[454, 495, 550, 509, 531, 530, 488, 523, 502, 480, 511, 498, 508, 548, 515, 505, 490, 511, 514, 527, 524, 498, 544, 558, 500, 515, 523, 496, 527, 515, 503, 497, 503, 497, 517, 520, 505, 494, 489, 494, 505, 492, 514, 510, 518, 499, 541, 484, 490, 534, 510, 531, 521, 558, 519, 538, 528, 511, 494, 526, 530, 502, 490, 515] payload_rows_per_dst=[5550, 5510, 5390, 5402, 5674, 5682, 5716, 5628] dup_rows_per_dst=[2610, 2560, 2334, 2398, 2906, 2796, 2712, 2668] combine_rows_per_src=[5550, 5510, 5390, 5402, 5674, 5682, 5716, 5628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=228168 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278934344 ingress_bytes=498907136 link_bytes=498907136 token_bytes=557640520 weight_movement_bytes=220200960 total_nvlink_bytes=777841480 weight_to_token_ratio=0.394880 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676815656e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278032384 dispatch_meta_bytes=228624 combine_payload_bytes=278032384 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278261008 ingress_bytes=498233344 link_bytes=498233344 token_bytes=556293392 weight_movement_bytes=220200960 total_nvlink_bytes=776494352 weight_to_token_ratio=0.395836 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.665097305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44640 payload_rows_remote=39066 dup_rows=20896 dup_rows_remote=18424 dedup_ratio=0.318848 router_tokens_per_expert=[512, 519, 511, 501, 546, 490, 537, 494, 514, 520, 536, 552, 503, 517, 508, 474, 511, 478, 504, 501, 524, 528, 500, 512, 554, 528, 506, 494, 473, 470, 518, 480, 514, 531, 513, 511, 525, 488, 527, 509, 516, 552, 483, 522, 512, 519, 520, 517, 511, 500, 512, 516, 510, 518, 513, 490, 507, 462, 559, 518, 516, 522, 519, 521] payload_rows_per_dst=[5542, 5632, 5498, 5574, 5546, 5598, 5634, 5616] dup_rows_per_dst=[2678, 2616, 2618, 2472, 2690, 2684, 2506, 2632] combine_rows_per_src=[5542, 5632, 5498, 5574, 5546, 5598, 5634, 5616] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278175744 dispatch_meta_bytes=227024 combine_payload_bytes=278175744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278402768 ingress_bytes=388276224 link_bytes=388276224 token_bytes=556578512 weight_movement_bytes=110100480 total_nvlink_bytes=666678992 weight_to_token_ratio=0.197817 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752762140e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44208 payload_rows_remote=38662 dup_rows=21328 dup_rows_remote=18672 dedup_ratio=0.325439 router_tokens_per_expert=[495, 528, 498, 527, 510, 525, 529, 508, 476, 487, 525, 500, 551, 528, 511, 523, 521, 509, 512, 542, 492, 511, 561, 511, 509, 525, 532, 510, 517, 514, 535, 494, 495, 482, 517, 509, 541, 498, 524, 523, 498, 475, 524, 503, 478, 488, 519, 506, 546, 533, 474, 512, 550, 495, 481, 531, 478, 523, 497, 512, 516, 488, 540, 496] payload_rows_per_dst=[5512, 5546, 5614, 5538, 5504, 5498, 5546, 5450] dup_rows_per_dst=[2728, 2656, 2704, 2734, 2674, 2484, 2698, 2650] combine_rows_per_src=[5512, 5546, 5614, 5538, 5504, 5498, 5546, 5450] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=38824 dup_rows=21006 dup_rows_remote=18192 dedup_ratio=0.320526 router_tokens_per_expert=[494, 537, 510, 491, 465, 490, 487, 510, 531, 503, 550, 530, 527, 514, 475, 526, 507, 517, 563, 521, 559, 438, 528, 513, 523, 518, 486, 494, 494, 530, 517, 522, 515, 511, 507, 485, 517, 540, 499, 504, 535, 497, 502, 505, 486, 489, 479, 512, 515, 498, 560, 535, 526, 571, 527, 528, 498, 517, 490, 503, 519, 504, 521, 503] payload_rows_per_dst=[5502, 5596, 5546, 5604, 5494, 5510, 5706, 5572] dup_rows_per_dst=[2466, 2716, 2746, 2564, 2662, 2500, 2814, 2538] combine_rows_per_src=[5502, 5596, 5546, 5604, 5494, 5510, 5706, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44640 payload_rows_remote=39066 dup_rows=20896 dup_rows_remote=18424 dedup_ratio=0.318848 router_tokens_per_expert=[512, 519, 511, 501, 546, 490, 537, 494, 514, 520, 536, 552, 503, 517, 508, 474, 511, 478, 504, 501, 524, 528, 500, 512, 554, 528, 506, 494, 473, 470, 518, 480, 514, 531, 513, 511, 525, 488, 527, 509, 516, 552, 483, 522, 512, 519, 520, 517, 511, 500, 512, 516, 510, 518, 513, 490, 507, 462, 559, 518, 516, 522, 519, 521] payload_rows_per_dst=[5542, 5632, 5498, 5574, 5546, 5598, 5634, 5616] dup_rows_per_dst=[2678, 2616, 2618, 2472, 2690, 2684, 2506, 2632] combine_rows_per_src=[5542, 5632, 5498, 5574, 5546, 5598, 5634, 5616] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279365632 dispatch_meta_bytes=230376 combine_payload_bytes=279365632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279596008 ingress_bytes=389466112 link_bytes=389466112 token_bytes=558961640 weight_movement_bytes=110100480 total_nvlink_bytes=669062120 weight_to_token_ratio=0.196973 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773456249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=228960 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279236192 ingress_bytes=389107712 link_bytes=389107712 token_bytes=558243424 weight_movement_bytes=110100480 total_nvlink_bytes=668343904 weight_to_token_ratio=0.197227 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767223084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229384 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279437320 ingress_bytes=389308416 link_bytes=389308416 token_bytes=558645256 weight_movement_bytes=110100480 total_nvlink_bytes=668745736 weight_to_token_ratio=0.197085 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770713657e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44612 payload_rows_remote=39088 dup_rows=20924 dup_rows_remote=18264 dedup_ratio=0.319275 router_tokens_per_expert=[485, 492, 555, 509, 518, 505, 465, 520, 519, 557, 537, 492, 493, 510, 481, 550, 533, 527, 523, 503, 510, 522, 519, 496, 498, 483, 537, 534, 497, 514, 501, 534, 509, 477, 507, 516, 509, 511, 522, 519, 546, 475, 524, 512, 495, 519, 520, 501, 480, 517, 516, 489, 508, 563, 498, 493, 480, 547, 481, 517, 517, 522, 542, 517] payload_rows_per_dst=[5544, 5594, 5624, 5582, 5558, 5524, 5582, 5604] dup_rows_per_dst=[2554, 2684, 2642, 2614, 2582, 2660, 2546, 2642] combine_rows_per_src=[5544, 5594, 5624, 5582, 5558, 5524, 5582, 5604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277129216 dispatch_meta_bytes=229336 combine_payload_bytes=277129216 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277358552 ingress_bytes=497330176 link_bytes=497330176 token_bytes=554487768 weight_movement_bytes=220200960 total_nvlink_bytes=774688728 weight_to_token_ratio=0.397125 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.649389728e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=228064 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278518496 ingress_bytes=608591872 link_bytes=608591872 token_bytes=556808928 weight_movement_bytes=330301440 total_nvlink_bytes=887110368 weight_to_token_ratio=0.593204 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058441362e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=229960 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280255048 ingress_bytes=500226048 link_bytes=500226048 token_bytes=560280136 weight_movement_bytes=220200960 total_nvlink_bytes=780481096 weight_to_token_ratio=0.393019 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.699753705e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38892 dup_rows=21042 dup_rows_remote=18304 dedup_ratio=0.321075 router_tokens_per_expert=[524, 485, 533, 466, 540, 499, 493, 519, 504, 515, 520, 512, 513, 518, 491, 478, 502, 519, 516, 510, 535, 541, 519, 528, 507, 487, 496, 517, 559, 540, 530, 497, 499, 525, 496, 517, 489, 515, 502, 509, 544, 486, 521, 528, 520, 504, 447, 537, 527, 538, 504, 530, 526, 545, 479, 528, 513, 496, 511, 512, 516, 474, 511, 506] payload_rows_per_dst=[5536, 5562, 5602, 5588, 5510, 5524, 5598, 5574] dup_rows_per_dst=[2582, 2540, 2738, 2678, 2594, 2650, 2756, 2504] combine_rows_per_src=[5536, 5562, 5602, 5588, 5510, 5524, 5598, 5574] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44432 payload_rows_remote=38696 dup_rows=21104 dup_rows_remote=18276 dedup_ratio=0.322021 router_tokens_per_expert=[527, 515, 512, 518, 474, 506, 518, 502, 562, 516, 483, 544, 518, 514, 493, 515, 511, 476, 505, 502, 534, 499, 522, 549, 530, 530, 524, 513, 498, 498, 559, 564, 480, 526, 515, 522, 525, 527, 521, 484, 465, 484, 505, 506, 510, 507, 500, 517, 503, 507, 524, 482, 539, 542, 498, 470, 511, 514, 523, 508, 522, 488, 514, 498] payload_rows_per_dst=[5736, 5626, 5568, 5746, 5584, 5434, 5214, 5524] dup_rows_per_dst=[2828, 2664, 2628, 2872, 2616, 2554, 2310, 2632] combine_rows_per_src=[5736, 5626, 5568, 5746, 5584, 5434, 5214, 5524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=228064 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278518496 ingress_bytes=608591872 link_bytes=608591872 token_bytes=556808928 weight_movement_bytes=330301440 total_nvlink_bytes=887110368 weight_to_token_ratio=0.593204 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058441362e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44208 payload_rows_remote=38662 dup_rows=21328 dup_rows_remote=18672 dedup_ratio=0.325439 router_tokens_per_expert=[495, 528, 498, 527, 510, 525, 529, 508, 476, 487, 525, 500, 551, 528, 511, 523, 521, 509, 512, 542, 492, 511, 561, 511, 509, 525, 532, 510, 517, 514, 535, 494, 495, 482, 517, 509, 541, 498, 524, 523, 498, 475, 524, 503, 478, 488, 519, 506, 546, 533, 474, 512, 550, 495, 481, 531, 478, 523, 497, 512, 516, 488, 540, 496] payload_rows_per_dst=[5512, 5546, 5614, 5538, 5504, 5498, 5546, 5450] dup_rows_per_dst=[2728, 2656, 2704, 2734, 2674, 2484, 2698, 2650] combine_rows_per_src=[5512, 5546, 5614, 5538, 5504, 5498, 5546, 5450] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280182784 dispatch_meta_bytes=229408 combine_payload_bytes=280182784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280412192 ingress_bytes=390283264 link_bytes=390283264 token_bytes=560594976 weight_movement_bytes=110100480 total_nvlink_bytes=670695456 weight_to_token_ratio=0.196399 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787667867e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38968 dup_rows=21094 dup_rows_remote=18488 dedup_ratio=0.321869 router_tokens_per_expert=[525, 546, 510, 504, 487, 498, 494, 504, 500, 535, 537, 502, 526, 509, 523, 477, 525, 500, 504, 509, 502, 527, 495, 533, 523, 485, 483, 539, 522, 556, 488, 492, 513, 529, 490, 497, 524, 521, 457, 509, 527, 475, 513, 503, 488, 496, 556, 521, 534, 504, 530, 507, 490, 545, 538, 520, 488, 503, 558, 512, 531, 494, 513, 522] payload_rows_per_dst=[5526, 5530, 5550, 5604, 5474, 5526, 5648, 5584] dup_rows_per_dst=[2610, 2688, 2640, 2572, 2606, 2632, 2688, 2658] combine_rows_per_src=[5526, 5530, 5550, 5604, 5474, 5526, 5648, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38924 dup_rows=20984 dup_rows_remote=18316 dedup_ratio=0.320190 router_tokens_per_expert=[454, 495, 550, 509, 531, 530, 488, 523, 502, 480, 511, 498, 508, 548, 515, 505, 490, 511, 514, 527, 524, 498, 544, 558, 500, 515, 523, 496, 527, 515, 503, 497, 503, 497, 517, 520, 505, 494, 489, 494, 505, 492, 514, 510, 518, 499, 541, 484, 490, 534, 510, 531, 521, 558, 519, 538, 528, 511, 494, 526, 530, 502, 490, 515] payload_rows_per_dst=[5550, 5510, 5390, 5402, 5674, 5682, 5716, 5628] dup_rows_per_dst=[2610, 2560, 2334, 2398, 2906, 2796, 2712, 2668] combine_rows_per_src=[5550, 5510, 5390, 5402, 5674, 5682, 5716, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44346 payload_rows_remote=38974 dup_rows=21190 dup_rows_remote=18620 dedup_ratio=0.323334 router_tokens_per_expert=[498, 539, 504, 492, 516, 479, 530, 526, 508, 500, 512, 499, 512, 486, 506, 494, 546, 511, 518, 542, 516, 525, 504, 480, 510, 513, 515, 502, 507, 519, 519, 500, 513, 526, 494, 461, 482, 514, 525, 456, 531, 521, 512, 558, 564, 477, 524, 532, 498, 527, 517, 534, 533, 530, 462, 498, 481, 511, 536, 523, 532, 547, 505, 516] payload_rows_per_dst=[5500, 5550, 5568, 5508, 5372, 5676, 5524, 5648] dup_rows_per_dst=[2668, 2484, 2716, 2662, 2570, 2762, 2674, 2654] combine_rows_per_src=[5500, 5550, 5568, 5508, 5372, 5676, 5524, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44432 payload_rows_remote=38696 dup_rows=21104 dup_rows_remote=18276 dedup_ratio=0.322021 router_tokens_per_expert=[527, 515, 512, 518, 474, 506, 518, 502, 562, 516, 483, 544, 518, 514, 493, 515, 511, 476, 505, 502, 534, 499, 522, 549, 530, 530, 524, 513, 498, 498, 559, 564, 480, 526, 515, 522, 525, 527, 521, 484, 465, 484, 505, 506, 510, 507, 500, 517, 503, 507, 524, 482, 539, 542, 498, 470, 511, 514, 523, 508, 522, 488, 514, 498] payload_rows_per_dst=[5736, 5626, 5568, 5746, 5584, 5434, 5214, 5524] dup_rows_per_dst=[2828, 2664, 2628, 2872, 2616, 2554, 2310, 2632] combine_rows_per_src=[5736, 5626, 5568, 5746, 5584, 5434, 5214, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38892 dup_rows=21042 dup_rows_remote=18304 dedup_ratio=0.321075 router_tokens_per_expert=[524, 485, 533, 466, 540, 499, 493, 519, 504, 515, 520, 512, 513, 518, 491, 478, 502, 519, 516, 510, 535, 541, 519, 528, 507, 487, 496, 517, 559, 540, 530, 497, 499, 525, 496, 517, 489, 515, 502, 509, 544, 486, 521, 528, 520, 504, 447, 537, 527, 538, 504, 530, 526, 545, 479, 528, 513, 496, 511, 512, 516, 474, 511, 506] payload_rows_per_dst=[5536, 5562, 5602, 5588, 5510, 5524, 5598, 5574] dup_rows_per_dst=[2582, 2540, 2738, 2678, 2594, 2650, 2756, 2504] combine_rows_per_src=[5536, 5562, 5602, 5588, 5510, 5524, 5598, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277129216 dispatch_meta_bytes=229336 combine_payload_bytes=277129216 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277358552 ingress_bytes=497330176 link_bytes=497330176 token_bytes=554487768 weight_movement_bytes=220200960 total_nvlink_bytes=774688728 weight_to_token_ratio=0.397125 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.649389728e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38968 dup_rows=20978 dup_rows_remote=18408 dedup_ratio=0.320099 router_tokens_per_expert=[511, 527, 530, 517, 493, 457, 517, 526, 475, 488, 521, 561, 523, 495, 515, 542, 521, 504, 484, 524, 506, 512, 524, 488, 525, 525, 572, 501, 520, 513, 542, 455, 511, 487, 473, 509, 510, 529, 502, 512, 505, 523, 523, 503, 519, 496, 530, 499, 504, 484, 530, 516, 517, 533, 524, 535, 505, 486, 500, 492, 511, 556, 515, 515] payload_rows_per_dst=[5496, 5572, 5530, 5564, 5534, 5592, 5680, 5590] dup_rows_per_dst=[2424, 2668, 2596, 2742, 2532, 2604, 2842, 2570] combine_rows_per_src=[5496, 5572, 5530, 5564, 5534, 5592, 5680, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=228784 combine_payload_bytes=278777856 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279006640 ingress_bytes=609079296 link_bytes=609079296 token_bytes=557784496 weight_movement_bytes=330301440 total_nvlink_bytes=888085936 weight_to_token_ratio=0.592167 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059289072e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38938 dup_rows=21088 dup_rows_remote=18398 dedup_ratio=0.321777 router_tokens_per_expert=[511, 578, 501, 517, 516, 491, 510, 527, 477, 513, 540, 504, 547, 475, 549, 524, 519, 527, 477, 526, 494, 512, 480, 508, 516, 488, 522, 549, 494, 521, 513, 497, 485, 490, 527, 505, 538, 476, 526, 501, 500, 525, 489, 537, 516, 509, 504, 493, 517, 472, 539, 509, 528, 538, 516, 536, 510, 510, 519, 512, 472, 491, 510, 545] payload_rows_per_dst=[5628, 5616, 5524, 5510, 5548, 5558, 5600, 5464] dup_rows_per_dst=[2674, 2642, 2562, 2690, 2548, 2588, 2710, 2674] combine_rows_per_src=[5628, 5616, 5524, 5510, 5548, 5558, 5600, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=228960 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279236192 ingress_bytes=389107712 link_bytes=389107712 token_bytes=558243424 weight_movement_bytes=110100480 total_nvlink_bytes=668343904 weight_to_token_ratio=0.197227 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767223084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277372928 dispatch_meta_bytes=227888 combine_payload_bytes=277372928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277600816 ingress_bytes=387473408 link_bytes=387473408 token_bytes=554973744 weight_movement_bytes=110100480 total_nvlink_bytes=665074224 weight_to_token_ratio=0.198389 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.738799849e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=39042 dup_rows=20934 dup_rows_remote=18432 dedup_ratio=0.319427 router_tokens_per_expert=[511, 530, 507, 495, 497, 500, 522, 477, 500, 515, 496, 501, 506, 493, 511, 509, 533, 525, 506, 499, 553, 502, 480, 490, 509, 538, 548, 543, 517, 503, 534, 514, 536, 506, 487, 533, 548, 499, 542, 525, 496, 501, 508, 532, 489, 544, 538, 508, 529, 508, 511, 508, 504, 491, 496, 495, 496, 496, 504, 534, 513, 508, 518, 501] payload_rows_per_dst=[5562, 5560, 5586, 5692, 5600, 5568, 5562, 5472] dup_rows_per_dst=[2516, 2502, 2590, 2720, 2752, 2664, 2522, 2668] combine_rows_per_src=[5562, 5560, 5586, 5692, 5600, 5568, 5562, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279365632 dispatch_meta_bytes=230376 combine_payload_bytes=279365632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279596008 ingress_bytes=389466112 link_bytes=389466112 token_bytes=558961640 weight_movement_bytes=110100480 total_nvlink_bytes=669062120 weight_to_token_ratio=0.196973 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773456249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=39010 dup_rows=21058 dup_rows_remote=18526 dedup_ratio=0.321320 router_tokens_per_expert=[512, 515, 526, 477, 521, 521, 492, 513, 550, 494, 528, 506, 553, 517, 474, 473, 532, 497, 537, 496, 493, 515, 493, 481, 491, 491, 548, 488, 540, 531, 521, 514, 511, 519, 477, 503, 531, 478, 575, 558, 522, 499, 514, 495, 546, 525, 538, 474, 496, 539, 487, 493, 494, 499, 518, 474, 539, 498, 525, 534, 533, 499, 517, 518] payload_rows_per_dst=[5594, 5534, 5482, 5608, 5612, 5564, 5468, 5616] dup_rows_per_dst=[2560, 2656, 2606, 2640, 2692, 2662, 2532, 2710] combine_rows_per_src=[5594, 5534, 5482, 5608, 5612, 5564, 5468, 5616] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38938 dup_rows=21088 dup_rows_remote=18398 dedup_ratio=0.321777 router_tokens_per_expert=[511, 578, 501, 517, 516, 491, 510, 527, 477, 513, 540, 504, 547, 475, 549, 524, 519, 527, 477, 526, 494, 512, 480, 508, 516, 488, 522, 549, 494, 521, 513, 497, 485, 490, 527, 505, 538, 476, 526, 501, 500, 525, 489, 537, 516, 509, 504, 493, 517, 472, 539, 509, 528, 538, 516, 536, 510, 510, 519, 512, 472, 491, 510, 545] payload_rows_per_dst=[5628, 5616, 5524, 5510, 5548, 5558, 5600, 5464] dup_rows_per_dst=[2674, 2642, 2562, 2690, 2548, 2588, 2710, 2674] combine_rows_per_src=[5628, 5616, 5524, 5510, 5548, 5558, 5600, 5464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229824 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279552448 ingress_bytes=389423104 link_bytes=389423104 token_bytes=558875072 weight_movement_bytes=110100480 total_nvlink_bytes=668975552 weight_to_token_ratio=0.197004 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772708270e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229504 combine_payload_bytes=279322624 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279552128 ingress_bytes=609624064 link_bytes=609624064 token_bytes=558874752 weight_movement_bytes=330301440 total_nvlink_bytes=889176192 weight_to_token_ratio=0.591012 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060236513e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=228784 combine_payload_bytes=278777856 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279006640 ingress_bytes=609079296 link_bytes=609079296 token_bytes=557784496 weight_movement_bytes=330301440 total_nvlink_bytes=888085936 weight_to_token_ratio=0.592167 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059289072e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38906 dup_rows=21000 dup_rows_remote=18344 dedup_ratio=0.320435 router_tokens_per_expert=[520, 494, 495, 514, 501, 505, 522, 537, 498, 495, 506, 528, 525, 501, 485, 512, 508, 528, 503, 507, 564, 520, 523, 498, 485, 504, 495, 533, 501, 493, 524, 527, 525, 527, 500, 517, 503, 519, 521, 514, 512, 527, 550, 545, 492, 506, 489, 522, 488, 517, 504, 528, 496, 490, 541, 488, 524, 520, 480, 541, 524, 520, 480, 507] payload_rows_per_dst=[5518, 5562, 5606, 5514, 5598, 5630, 5528, 5580] dup_rows_per_dst=[2658, 2538, 2696, 2610, 2654, 2656, 2576, 2612] combine_rows_per_src=[5518, 5562, 5606, 5514, 5598, 5630, 5528, 5580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=230144 combine_payload_bytes=279623680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279853824 ingress_bytes=389724160 link_bytes=389724160 token_bytes=559477504 weight_movement_bytes=110100480 total_nvlink_bytes=669577984 weight_to_token_ratio=0.196792 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777944128e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279853056 dispatch_meta_bytes=229896 combine_payload_bytes=279853056 prefetch_bytes=0 reduce_bytes=0 egress_bytes=280082952 ingress_bytes=279853056 link_bytes=280082952 token_bytes=559936008 weight_movement_bytes=0 total_nvlink_bytes=559936008 weight_to_token_ratio=0.000000 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=4.871103193e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229344 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279336928 ingress_bytes=499308544 link_bytes=499308544 token_bytes=558444512 weight_movement_bytes=220200960 total_nvlink_bytes=778645472 weight_to_token_ratio=0.394311 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683796802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5579598640 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=2790945584 ingress_bytes=4440160256 link_bytes=4440160256 total_nvlink_bytes=7231105840 weight_to_token_ratio=0.295990 prefetch_share_of_ingress=0.247965 dedup_ratio=0.321536 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.722168966e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5570419104 prefetch_bytes=770703360 reduce_bytes=1101004800 weight_movement_bytes=1871708160 egress_bytes=2786353568 ingress_bytes=4655773696 link_bytes=4655773696 total_nvlink_bytes=7442127264 weight_to_token_ratio=0.336008 prefetch_share_of_ingress=0.165537 dedup_ratio=0.321783 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=10 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.097156201e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44634 payload_rows_remote=38976 dup_rows=20902 dup_rows_remote=18220 dedup_ratio=0.318939 router_tokens_per_expert=[548, 511, 536, 497, 509, 521, 542, 490, 495, 546, 510, 530, 502, 545, 495, 516, 498, 494, 502, 500, 513, 489, 521, 498, 533, 524, 508, 506, 486, 521, 500, 529, 508, 497, 527, 555, 517, 477, 470, 500, 493, 506, 513, 521, 546, 501, 482, 538, 529, 487, 535, 507, 524, 515, 522, 475, 546, 516, 478, 528, 504, 521, 496, 519] payload_rows_per_dst=[5658, 5602, 5706, 5398, 5608, 5496, 5592, 5574] dup_rows_per_dst=[2682, 2676, 2796, 2420, 2658, 2554, 2596, 2520] combine_rows_per_src=[5658, 5602, 5706, 5398, 5608, 5496, 5592, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38880 dup_rows=21166 dup_rows_remote=18590 dedup_ratio=0.322968 router_tokens_per_expert=[509, 552, 541, 534, 525, 545, 492, 517, 511, 499, 516, 525, 512, 480, 537, 515, 496, 492, 504, 527, 492, 495, 529, 498, 508, 520, 513, 499, 507, 520, 545, 514, 516, 475, 524, 523, 464, 515, 531, 512, 477, 505, 502, 513, 543, 537, 525, 519, 508, 500, 512, 473, 501, 471, 522, 509, 459, 493, 532, 539, 511, 513, 573, 502] payload_rows_per_dst=[5596, 5598, 5490, 5558, 5516, 5536, 5464, 5612] dup_rows_per_dst=[2834, 2592, 2576, 2694, 2604, 2706, 2528, 2632] combine_rows_per_src=[5596, 5598, 5490, 5558, 5516, 5536, 5464, 5612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5580144008 prefetch_bytes=880803840 reduce_bytes=990904320 weight_movement_bytes=1871708160 egress_bytes=2791218568 ingress_bytes=4660633600 link_bytes=4660633600 total_nvlink_bytes=7451852168 weight_to_token_ratio=0.335423 prefetch_share_of_ingress=0.188988 dedup_ratio=0.321182 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=9 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.105608373e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5585162152 prefetch_bytes=880803840 reduce_bytes=990904320 weight_movement_bytes=1871708160 egress_bytes=2793727912 ingress_bytes=4663142400 link_bytes=4663372296 total_nvlink_bytes=7456870312 weight_to_token_ratio=0.335122 prefetch_share_of_ingress=0.188886 dedup_ratio=0.320901 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=9 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.110371416e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=230144 combine_payload_bytes=279623680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279853824 ingress_bytes=389724160 link_bytes=389724160 token_bytes=559477504 weight_movement_bytes=110100480 total_nvlink_bytes=669577984 weight_to_token_ratio=0.196792 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777944128e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=39042 dup_rows=20934 dup_rows_remote=18432 dedup_ratio=0.319427 router_tokens_per_expert=[511, 530, 507, 495, 497, 500, 522, 477, 500, 515, 496, 501, 506, 493, 511, 509, 533, 525, 506, 499, 553, 502, 480, 490, 509, 538, 548, 543, 517, 503, 534, 514, 536, 506, 487, 533, 548, 499, 542, 525, 496, 501, 508, 532, 489, 544, 538, 508, 529, 508, 511, 508, 504, 491, 496, 495, 496, 496, 504, 534, 513, 508, 518, 501] payload_rows_per_dst=[5562, 5560, 5586, 5692, 5600, 5568, 5562, 5472] dup_rows_per_dst=[2516, 2502, 2590, 2720, 2752, 2664, 2522, 2668] combine_rows_per_src=[5562, 5560, 5586, 5692, 5600, 5568, 5562, 5472] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=229000 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279107208 ingress_bytes=499079168 link_bytes=499079168 token_bytes=557985416 weight_movement_bytes=220200960 total_nvlink_bytes=778186376 weight_to_token_ratio=0.394636 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679807576e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5581835752 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2792064488 ingress_bytes=4771579904 link_bytes=4771579904 total_nvlink_bytes=7563644392 weight_to_token_ratio=0.355046 prefetch_share_of_ingress=0.207668 dedup_ratio=0.321082 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.298562243e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38968 dup_rows=20978 dup_rows_remote=18408 dedup_ratio=0.320099 router_tokens_per_expert=[511, 527, 530, 517, 493, 457, 517, 526, 475, 488, 521, 561, 523, 495, 515, 542, 521, 504, 484, 524, 506, 512, 524, 488, 525, 525, 572, 501, 520, 513, 542, 455, 511, 487, 473, 509, 510, 529, 502, 512, 505, 523, 523, 503, 519, 496, 530, 499, 504, 484, 530, 516, 517, 533, 524, 535, 505, 486, 500, 492, 511, 556, 515, 515] payload_rows_per_dst=[5496, 5572, 5530, 5564, 5534, 5592, 5680, 5590] dup_rows_per_dst=[2424, 2668, 2596, 2742, 2532, 2604, 2842, 2570] combine_rows_per_src=[5496, 5572, 5530, 5564, 5534, 5592, 5680, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44634 payload_rows_remote=38976 dup_rows=20902 dup_rows_remote=18220 dedup_ratio=0.318939 router_tokens_per_expert=[548, 511, 536, 497, 509, 521, 542, 490, 495, 546, 510, 530, 502, 545, 495, 516, 498, 494, 502, 500, 513, 489, 521, 498, 533, 524, 508, 506, 486, 521, 500, 529, 508, 497, 527, 555, 517, 477, 470, 500, 493, 506, 513, 521, 546, 501, 482, 538, 529, 487, 535, 507, 524, 515, 522, 475, 546, 516, 478, 528, 504, 521, 496, 519] payload_rows_per_dst=[5658, 5602, 5706, 5398, 5608, 5496, 5592, 5574] dup_rows_per_dst=[2682, 2676, 2796, 2420, 2658, 2554, 2596, 2520] combine_rows_per_src=[5658, 5602, 5706, 5398, 5608, 5496, 5592, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38968 dup_rows=21094 dup_rows_remote=18488 dedup_ratio=0.321869 router_tokens_per_expert=[525, 546, 510, 504, 487, 498, 494, 504, 500, 535, 537, 502, 526, 509, 523, 477, 525, 500, 504, 509, 502, 527, 495, 533, 523, 485, 483, 539, 522, 556, 488, 492, 513, 529, 490, 497, 524, 521, 457, 509, 527, 475, 513, 503, 488, 496, 556, 521, 534, 504, 530, 507, 490, 545, 538, 520, 488, 503, 558, 512, 531, 494, 513, 522] payload_rows_per_dst=[5526, 5530, 5550, 5604, 5474, 5526, 5648, 5584] dup_rows_per_dst=[2610, 2688, 2640, 2572, 2606, 2632, 2688, 2658] combine_rows_per_src=[5526, 5530, 5550, 5604, 5474, 5526, 5648, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5580144008 prefetch_bytes=880803840 reduce_bytes=990904320 weight_movement_bytes=1871708160 egress_bytes=2791218568 ingress_bytes=4660633600 link_bytes=4660633600 total_nvlink_bytes=7451852168 weight_to_token_ratio=0.335423 prefetch_share_of_ingress=0.188988 dedup_ratio=0.321182 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=9 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.105608373e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5579340840 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2790816808 ingress_bytes=4660232192 link_bytes=4660232192 total_nvlink_bytes=7451049000 weight_to_token_ratio=0.335471 prefetch_share_of_ingress=0.236255 dedup_ratio=0.321558 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.104910258e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279853056 dispatch_meta_bytes=229896 combine_payload_bytes=279853056 prefetch_bytes=0 reduce_bytes=0 egress_bytes=280082952 ingress_bytes=279853056 link_bytes=280082952 token_bytes=559936008 weight_movement_bytes=0 total_nvlink_bytes=559936008 weight_to_token_ratio=0.000000 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=4.871103193e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38880 dup_rows=21166 dup_rows_remote=18590 dedup_ratio=0.322968 router_tokens_per_expert=[509, 552, 541, 534, 525, 545, 492, 517, 511, 499, 516, 525, 512, 480, 537, 515, 496, 492, 504, 527, 492, 495, 529, 498, 508, 520, 513, 499, 507, 520, 545, 514, 516, 475, 524, 523, 464, 515, 531, 512, 477, 505, 502, 513, 543, 537, 525, 519, 508, 500, 512, 473, 501, 471, 522, 509, 459, 493, 532, 539, 511, 513, 573, 502] payload_rows_per_dst=[5596, 5598, 5490, 5558, 5516, 5536, 5464, 5612] dup_rows_per_dst=[2834, 2592, 2576, 2694, 2604, 2706, 2528, 2632] combine_rows_per_src=[5596, 5598, 5490, 5558, 5516, 5536, 5464, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229504 combine_payload_bytes=279322624 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279552128 ingress_bytes=609624064 link_bytes=609624064 token_bytes=558874752 weight_movement_bytes=330301440 total_nvlink_bytes=889176192 weight_to_token_ratio=0.591012 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060236513e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=228784 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279608752 ingress_bytes=499580928 link_bytes=499580928 token_bytes=558988720 weight_movement_bytes=220200960 total_nvlink_bytes=779189680 weight_to_token_ratio=0.393927 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688534007e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229824 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279552448 ingress_bytes=389423104 link_bytes=389423104 token_bytes=558875072 weight_movement_bytes=110100480 total_nvlink_bytes=668975552 weight_to_token_ratio=0.197004 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772708270e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5585162152 prefetch_bytes=880803840 reduce_bytes=990904320 weight_movement_bytes=1871708160 egress_bytes=2793727912 ingress_bytes=4663142400 link_bytes=4663372296 total_nvlink_bytes=7456870312 weight_to_token_ratio=0.335122 prefetch_share_of_ingress=0.188886 dedup_ratio=0.320901 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=9 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.110371416e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229880 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278921720 ingress_bytes=388792320 link_bytes=388792320 token_bytes=557613560 weight_movement_bytes=110100480 total_nvlink_bytes=667714040 weight_to_token_ratio=0.197449 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761737898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5579340840 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2790816808 ingress_bytes=4660232192 link_bytes=4660232192 total_nvlink_bytes=7451049000 weight_to_token_ratio=0.335471 prefetch_share_of_ingress=0.236255 dedup_ratio=0.321558 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.104910258e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5570419104 prefetch_bytes=770703360 reduce_bytes=1101004800 weight_movement_bytes=1871708160 egress_bytes=2786353568 ingress_bytes=4655773696 link_bytes=4655773696 total_nvlink_bytes=7442127264 weight_to_token_ratio=0.336008 prefetch_share_of_ingress=0.165537 dedup_ratio=0.321783 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=10 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.097156201e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5579598640 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=2790945584 ingress_bytes=4440160256 link_bytes=4440160256 total_nvlink_bytes=7231105840 weight_to_token_ratio=0.295990 prefetch_share_of_ingress=0.247965 dedup_ratio=0.321536 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.722168966e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5567864416 prefetch_bytes=770703360 reduce_bytes=1541406720 weight_movement_bytes=2312110080 egress_bytes=2785074784 ingress_bytes=5094899712 link_bytes=5094899712 total_nvlink_bytes=7879974496 weight_to_token_ratio=0.415260 prefetch_share_of_ingress=0.151270 dedup_ratio=0.321497 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=14 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.860868566e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5581521344 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=2791907776 ingress_bytes=4551221248 link_bytes=4551221248 total_nvlink_bytes=7343129024 weight_to_token_ratio=0.315614 prefetch_share_of_ingress=0.241914 dedup_ratio=0.321448 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.915322298e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none - [2026-08-26 16:58:52] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 365.9 | throughput per GPU (TFLOP/s/GPU): 337.4 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.115080E+01 | loss scale: 1.0 | grad norm: 107.071 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=38964 dup_rows=21036 dup_rows_remote=18400 dedup_ratio=0.320984 router_tokens_per_expert=[510, 527, 526, 501, 504, 493, 505, 550, 542, 519, 505, 475, 512, 506, 520, 539, 555, 509, 514, 531, 496, 508, 512, 510, 540, 514, 513, 466, 491, 505, 509, 503, 515, 505, 520, 454, 486, 547, 553, 506, 524, 489, 523, 499, 498, 498, 548, 510, 527, 547, 520, 510, 529, 493, 483, 503, 506, 498, 504, 488, 539, 525, 492, 519] payload_rows_per_dst=[5554, 5634, 5558, 5588, 5536, 5524, 5604, 5502] dup_rows_per_dst=[2678, 2602, 2712, 2494, 2636, 2654, 2620, 2640] combine_rows_per_src=[5554, 5634, 5558, 5588, 5536, 5524, 5604, 5502] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38934 dup_rows=21082 dup_rows_remote=18408 dedup_ratio=0.321686 router_tokens_per_expert=[509, 503, 521, 494, 512, 507, 518, 503, 502, 499, 517, 494, 528, 519, 505, 505, 497, 544, 509, 483, 499, 504, 524, 530, 529, 534, 521, 509, 487, 499, 548, 510, 526, 493, 528, 511, 516, 479, 543, 504, 545, 559, 526, 463, 497, 523, 519, 465, 508, 500, 547, 531, 537, 532, 494, 479, 508, 554, 469, 516, 491, 473, 544, 525] payload_rows_per_dst=[5598, 5528, 5586, 5562, 5496, 5520, 5622, 5542] dup_rows_per_dst=[2536, 2610, 2594, 2712, 2704, 2674, 2634, 2618] combine_rows_per_src=[5598, 5528, 5586, 5562, 5496, 5520, 5622, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44384 payload_rows_remote=38810 dup_rows=21152 dup_rows_remote=18516 dedup_ratio=0.322754 router_tokens_per_expert=[523, 526, 517, 493, 551, 525, 505, 523, 509, 508, 508, 505, 536, 521, 482, 536, 525, 513, 525, 499, 543, 520, 548, 472, 471, 527, 496, 501, 524, 512, 506, 546, 513, 530, 492, 438, 538, 523, 526, 545, 523, 523, 466, 525, 499, 499, 539, 493, 504, 512, 478, 510, 499, 511, 457, 530, 510, 499, 498, 515, 504, 541, 527, 505] payload_rows_per_dst=[5576, 5574, 5606, 5570, 5514, 5498, 5484, 5562] dup_rows_per_dst=[2750, 2636, 2684, 2596, 2696, 2636, 2518, 2636] combine_rows_per_src=[5576, 5574, 5606, 5570, 5514, 5498, 5484, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44580 payload_rows_remote=38946 dup_rows=20956 dup_rows_remote=18134 dedup_ratio=0.319763 router_tokens_per_expert=[531, 461, 498, 498, 518, 496, 507, 534, 477, 494, 540, 548, 512, 516, 485, 516, 524, 510, 537, 504, 513, 516, 474, 518, 495, 530, 496, 524, 518, 509, 520, 547, 520, 544, 514, 519, 506, 509, 526, 499, 511, 509, 489, 509, 524, 508, 486, 507, 538, 495, 537, 527, 501, 505, 484, 507, 512, 510, 535, 510, 514, 537, 502, 508] payload_rows_per_dst=[5564, 5520, 5506, 5618, 5612, 5442, 5684, 5634] dup_rows_per_dst=[2522, 2458, 2486, 2756, 2662, 2440, 2810, 2822] combine_rows_per_src=[5564, 5520, 5506, 5618, 5612, 5442, 5684, 5634] + [2026-08-26 18:10:57] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 363.9 | throughput per GPU (TFLOP/s/GPU): 339.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.115797E+01 | loss scale: 1.0 | grad norm: 107.147 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38802 dup_rows=21114 dup_rows_remote=18474 dedup_ratio=0.322174 router_tokens_per_expert=[518, 490, 482, 520, 542, 511, 469, 491, 505, 538, 510, 512, 499, 555, 557, 527, 492, 510, 534, 546, 495, 524, 497, 525, 482, 538, 471, 549, 510, 535, 507, 538, 519, 518, 529, 518, 504, 478, 529, 498, 510, 522, 525, 507, 506, 476, 528, 496, 491, 500, 481, 539, 512, 514, 463, 524, 546, 494, 530, 527, 538, 521, 477, 469] payload_rows_per_dst=[5494, 5640, 5528, 5620, 5534, 5516, 5536, 5554] dup_rows_per_dst=[2552, 2766, 2718, 2640, 2652, 2624, 2512, 2650] combine_rows_per_src=[5494, 5640, 5528, 5620, 5534, 5516, 5536, 5554] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=39142 dup_rows=20936 dup_rows_remote=18318 dedup_ratio=0.319458 router_tokens_per_expert=[489, 528, 464, 486, 523, 521, 521, 484, 506, 508, 548, 469, 537, 526, 525, 535, 497, 500, 489, 534, 541, 492, 507, 511, 505, 506, 503, 511, 515, 531, 538, 495, 544, 510, 530, 503, 511, 529, 504, 540, 553, 549, 497, 491, 532, 515, 557, 512, 476, 528, 473, 527, 522, 512, 496, 504, 472, 523, 482, 498, 481, 502, 533, 517] payload_rows_per_dst=[5500, 5570, 5572, 5616, 5634, 5724, 5458, 5526] dup_rows_per_dst=[2532, 2738, 2570, 2592, 2708, 2688, 2618, 2490] combine_rows_per_src=[5500, 5570, 5572, 5616, 5634, 5724, 5458, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38934 dup_rows=21082 dup_rows_remote=18408 dedup_ratio=0.321686 router_tokens_per_expert=[509, 503, 521, 494, 512, 507, 518, 503, 502, 499, 517, 494, 528, 519, 505, 505, 497, 544, 509, 483, 499, 504, 524, 530, 529, 534, 521, 509, 487, 499, 548, 510, 526, 493, 528, 511, 516, 479, 543, 504, 545, 559, 526, 463, 497, 523, 519, 465, 508, 500, 547, 531, 537, 532, 494, 479, 508, 554, 469, 516, 491, 473, 544, 525] payload_rows_per_dst=[5598, 5528, 5586, 5562, 5496, 5520, 5622, 5542] dup_rows_per_dst=[2536, 2610, 2594, 2712, 2704, 2674, 2634, 2618] combine_rows_per_src=[5598, 5528, 5586, 5562, 5496, 5520, 5622, 5542] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44496 payload_rows_remote=38836 dup_rows=21040 dup_rows_remote=18434 dedup_ratio=0.321045 router_tokens_per_expert=[515, 570, 520, 531, 522, 503, 514, 488, 526, 520, 493, 496, 492, 542, 474, 500, 493, 498, 550, 530, 502, 530, 500, 530, 507, 517, 503, 483, 515, 512, 520, 498, 500, 528, 514, 499, 541, 508, 489, 494, 531, 487, 525, 518, 520, 529, 521, 499, 560, 491, 509, 513, 521, 496, 547, 490, 516, 478, 496, 511, 519, 500, 522, 502] payload_rows_per_dst=[5646, 5516, 5660, 5552, 5558, 5560, 5534, 5470] dup_rows_per_dst=[2680, 2570, 2606, 2558, 2588, 2700, 2720, 2618] combine_rows_per_src=[5646, 5516, 5660, 5552, 5558, 5560, 5534, 5470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44580 payload_rows_remote=38946 dup_rows=20956 dup_rows_remote=18134 dedup_ratio=0.319763 router_tokens_per_expert=[531, 461, 498, 498, 518, 496, 507, 534, 477, 494, 540, 548, 512, 516, 485, 516, 524, 510, 537, 504, 513, 516, 474, 518, 495, 530, 496, 524, 518, 509, 520, 547, 520, 544, 514, 519, 506, 509, 526, 499, 511, 509, 489, 509, 524, 508, 486, 507, 538, 495, 537, 527, 501, 505, 484, 507, 512, 510, 535, 510, 514, 537, 502, 508] payload_rows_per_dst=[5564, 5520, 5506, 5618, 5612, 5442, 5684, 5634] dup_rows_per_dst=[2522, 2458, 2486, 2756, 2662, 2440, 2810, 2822] combine_rows_per_src=[5564, 5520, 5506, 5618, 5612, 5442, 5684, 5634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44384 payload_rows_remote=38810 dup_rows=21152 dup_rows_remote=18516 dedup_ratio=0.322754 router_tokens_per_expert=[523, 526, 517, 493, 551, 525, 505, 523, 509, 508, 508, 505, 536, 521, 482, 536, 525, 513, 525, 499, 543, 520, 548, 472, 471, 527, 496, 501, 524, 512, 506, 546, 513, 530, 492, 438, 538, 523, 526, 545, 523, 523, 466, 525, 499, 499, 539, 493, 504, 512, 478, 510, 499, 511, 457, 530, 510, 499, 498, 515, 504, 541, 527, 505] payload_rows_per_dst=[5576, 5574, 5606, 5570, 5514, 5498, 5484, 5562] dup_rows_per_dst=[2750, 2636, 2684, 2596, 2696, 2636, 2518, 2636] combine_rows_per_src=[5576, 5574, 5606, 5570, 5514, 5498, 5484, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=38964 dup_rows=21036 dup_rows_remote=18400 dedup_ratio=0.320984 router_tokens_per_expert=[510, 527, 526, 501, 504, 493, 505, 550, 542, 519, 505, 475, 512, 506, 520, 539, 555, 509, 514, 531, 496, 508, 512, 510, 540, 514, 513, 466, 491, 505, 509, 503, 515, 505, 520, 454, 486, 547, 553, 506, 524, 489, 523, 499, 498, 498, 548, 510, 527, 547, 520, 510, 529, 493, 483, 503, 506, 498, 504, 488, 539, 525, 492, 519] payload_rows_per_dst=[5554, 5634, 5558, 5588, 5536, 5524, 5604, 5502] dup_rows_per_dst=[2678, 2602, 2712, 2494, 2636, 2654, 2620, 2640] combine_rows_per_src=[5554, 5634, 5558, 5588, 5536, 5524, 5604, 5502] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38918 dup_rows=21026 dup_rows_remote=18264 dedup_ratio=0.320831 router_tokens_per_expert=[544, 477, 537, 485, 526, 521, 483, 524, 508, 554, 494, 517, 514, 491, 488, 513, 515, 530, 511, 491, 493, 490, 532, 509, 488, 479, 502, 505, 513, 526, 471, 547, 513, 529, 468, 493, 497, 523, 557, 515, 505, 514, 510, 536, 526, 524, 538, 515, 519, 532, 512, 537, 499, 495, 500, 497, 542, 521, 509, 536, 543, 498, 489, 498] payload_rows_per_dst=[5592, 5494, 5584, 5524, 5428, 5688, 5588, 5612] dup_rows_per_dst=[2762, 2680, 2566, 2538, 2578, 2648, 2594, 2660] combine_rows_per_src=[5592, 5494, 5584, 5524, 5428, 5688, 5588, 5612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229456 combine_payload_bytes=279293952 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279523408 ingress_bytes=609595392 link_bytes=609595392 token_bytes=558817360 weight_movement_bytes=330301440 total_nvlink_bytes=889118800 weight_to_token_ratio=0.591072 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060186648e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=229368 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279308280 ingress_bytes=499279872 link_bytes=499279872 token_bytes=558387192 weight_movement_bytes=220200960 total_nvlink_bytes=778588152 weight_to_token_ratio=0.394352 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683298148e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278190080 dispatch_meta_bytes=229304 combine_payload_bytes=278190080 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278419384 ingress_bytes=608491520 link_bytes=608491520 token_bytes=556609464 weight_movement_bytes=330301440 total_nvlink_bytes=886910904 weight_to_token_ratio=0.593417 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058266833e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=228320 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279393248 ingress_bytes=389265408 link_bytes=389265408 token_bytes=558558176 weight_movement_bytes=110100480 total_nvlink_bytes=668658656 weight_to_token_ratio=0.197116 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769965677e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38918 dup_rows=21026 dup_rows_remote=18264 dedup_ratio=0.320831 router_tokens_per_expert=[544, 477, 537, 485, 526, 521, 483, 524, 508, 554, 494, 517, 514, 491, 488, 513, 515, 530, 511, 491, 493, 490, 532, 509, 488, 479, 502, 505, 513, 526, 471, 547, 513, 529, 468, 493, 497, 523, 557, 515, 505, 514, 510, 536, 526, 524, 538, 515, 519, 532, 512, 537, 499, 495, 500, 497, 542, 521, 509, 536, 543, 498, 489, 498] payload_rows_per_dst=[5592, 5494, 5584, 5524, 5428, 5688, 5588, 5612] dup_rows_per_dst=[2762, 2680, 2566, 2538, 2578, 2648, 2594, 2660] combine_rows_per_src=[5592, 5494, 5584, 5524, 5428, 5688, 5588, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278132736 dispatch_meta_bytes=229104 combine_payload_bytes=278132736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278361840 ingress_bytes=388233216 link_bytes=388233216 token_bytes=556494576 weight_movement_bytes=110100480 total_nvlink_bytes=666595056 weight_to_token_ratio=0.197846 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752014160e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280569856 dispatch_meta_bytes=229840 combine_payload_bytes=280569856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280799696 ingress_bytes=390670336 link_bytes=390670336 token_bytes=561369552 weight_movement_bytes=110100480 total_nvlink_bytes=671470032 weight_to_token_ratio=0.196128 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794399685e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=229368 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279308280 ingress_bytes=499279872 link_bytes=499279872 token_bytes=558387192 weight_movement_bytes=220200960 total_nvlink_bytes=778588152 weight_to_token_ratio=0.394352 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683298148e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278376448 dispatch_meta_bytes=229080 combine_payload_bytes=278376448 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278605528 ingress_bytes=498577408 link_bytes=498577408 token_bytes=556981976 weight_movement_bytes=220200960 total_nvlink_bytes=777182936 weight_to_token_ratio=0.395347 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671081144e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=228320 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279393248 ingress_bytes=389265408 link_bytes=389265408 token_bytes=558558176 weight_movement_bytes=110100480 total_nvlink_bytes=668658656 weight_to_token_ratio=0.197116 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769965677e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278190080 dispatch_meta_bytes=229304 combine_payload_bytes=278190080 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278419384 ingress_bytes=608491520 link_bytes=608491520 token_bytes=556609464 weight_movement_bytes=330301440 total_nvlink_bytes=886910904 weight_to_token_ratio=0.593417 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058266833e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229456 combine_payload_bytes=279293952 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279523408 ingress_bytes=609595392 link_bytes=609595392 token_bytes=558817360 weight_movement_bytes=330301440 total_nvlink_bytes=889118800 weight_to_token_ratio=0.591072 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060186648e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=228728 combine_payload_bytes=278964224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279192952 ingress_bytes=389064704 link_bytes=389064704 token_bytes=558157176 weight_movement_bytes=110100480 total_nvlink_bytes=668257656 weight_to_token_ratio=0.197257 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766475104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38912 dup_rows=21010 dup_rows_remote=18420 dedup_ratio=0.320587 router_tokens_per_expert=[488, 553, 507, 476, 510, 488, 504, 487, 553, 528, 520, 530, 495, 480, 492, 515, 493, 499, 526, 469, 507, 537, 515, 507, 522, 528, 535, 531, 517, 507, 529, 524, 545, 519, 542, 482, 502, 504, 503, 505, 509, 546, 472, 516, 477, 492, 520, 515, 491, 533, 520, 540, 509, 491, 501, 491, 527, 550, 539, 517, 495, 495, 535, 513] payload_rows_per_dst=[5522, 5652, 5466, 5602, 5614, 5522, 5552, 5596] dup_rows_per_dst=[2504, 2574, 2640, 2784, 2590, 2572, 2600, 2746] combine_rows_per_src=[5522, 5652, 5466, 5602, 5614, 5522, 5552, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44334 payload_rows_remote=38738 dup_rows=21202 dup_rows_remote=18542 dedup_ratio=0.323517 router_tokens_per_expert=[502, 516, 530, 518, 488, 505, 494, 502, 508, 519, 492, 544, 519, 495, 518, 503, 504, 499, 514, 470, 518, 521, 547, 530, 531, 525, 514, 509, 492, 512, 548, 497, 508, 562, 529, 494, 531, 510, 504, 485, 525, 507, 502, 533, 551, 508, 534, 503, 532, 489, 499, 477, 475, 517, 520, 479, 505, 506, 470, 527, 552, 524, 515, 511] payload_rows_per_dst=[5516, 5540, 5536, 5596, 5594, 5578, 5502, 5472] dup_rows_per_dst=[2594, 2656, 2670, 2660, 2652, 2748, 2474, 2748] combine_rows_per_src=[5516, 5540, 5536, 5596, 5594, 5578, 5502, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38916 dup_rows=21116 dup_rows_remote=18480 dedup_ratio=0.322205 router_tokens_per_expert=[533, 497, 527, 515, 539, 484, 506, 519, 505, 562, 471, 486, 543, 490, 465, 497, 529, 485, 494, 497, 530, 546, 518, 460, 521, 498, 548, 527, 476, 560, 505, 546, 501, 526, 517, 512, 505, 500, 515, 514, 510, 504, 512, 520, 516, 503, 523, 522, 512, 513, 534, 494, 498, 503, 497, 519, 492, 502, 519, 486, 511, 548, 513, 548] payload_rows_per_dst=[5548, 5524, 5558, 5602, 5496, 5592, 5504, 5596] dup_rows_per_dst=[2692, 2514, 2560, 2760, 2684, 2628, 2636, 2642] combine_rows_per_src=[5548, 5524, 5558, 5602, 5496, 5592, 5504, 5596] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38790 dup_rows=21064 dup_rows_remote=18460 dedup_ratio=0.321411 router_tokens_per_expert=[517, 541, 507, 528, 547, 505, 504, 520, 478, 500, 550, 544, 498, 523, 501, 491, 520, 477, 471, 474, 496, 474, 518, 537, 513, 512, 524, 485, 530, 513, 480, 503, 510, 459, 450, 522, 499, 514, 523, 516, 514, 558, 518, 501, 522, 483, 549, 498, 542, 504, 504, 546, 480, 542, 506, 499, 548, 553, 498, 537, 553, 495, 511, 533] payload_rows_per_dst=[5598, 5502, 5478, 5506, 5454, 5682, 5562, 5690] dup_rows_per_dst=[2740, 2668, 2456, 2614, 2532, 2604, 2684, 2766] combine_rows_per_src=[5598, 5502, 5478, 5506, 5454, 5682, 5562, 5690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38824 dup_rows=21116 dup_rows_remote=18414 dedup_ratio=0.322205 router_tokens_per_expert=[486, 499, 534, 496, 505, 516, 529, 512, 531, 497, 511, 519, 535, 557, 484, 533, 530, 514, 499, 540, 504, 550, 509, 503, 514, 502, 553, 518, 496, 530, 509, 516, 475, 519, 531, 500, 548, 531, 511, 505, 507, 489, 509, 499, 489, 477, 530, 497, 536, 508, 503, 509, 498, 496, 530, 500, 496, 522, 504, 512, 493, 532, 508, 473] payload_rows_per_dst=[5532, 5598, 5596, 5558, 5558, 5482, 5598, 5498] dup_rows_per_dst=[2622, 2736, 2702, 2718, 2682, 2512, 2562, 2582] combine_rows_per_src=[5532, 5598, 5596, 5558, 5558, 5482, 5598, 5498] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44730 payload_rows_remote=39114 dup_rows=20806 dup_rows_remote=18124 dedup_ratio=0.317474 router_tokens_per_expert=[509, 512, 508, 502, 530, 542, 527, 504, 503, 523, 484, 488, 512, 497, 512, 488, 495, 490, 508, 509, 564, 508, 484, 487, 534, 523, 520, 505, 517, 526, 532, 462, 511, 521, 497, 523, 479, 520, 520, 515, 477, 517, 549, 520, 528, 488, 533, 529, 527, 477, 512, 519, 527, 519, 508, 502, 516, 537, 517, 530, 489, 510, 511, 535] payload_rows_per_dst=[5614, 5456, 5552, 5654, 5662, 5630, 5546, 5616] dup_rows_per_dst=[2654, 2502, 2538, 2584, 2510, 2652, 2684, 2682] combine_rows_per_src=[5614, 5456, 5552, 5654, 5662, 5630, 5546, 5616] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44632 payload_rows_remote=39046 dup_rows=20904 dup_rows_remote=18186 dedup_ratio=0.318970 router_tokens_per_expert=[497, 534, 526, 536, 513, 521, 474, 510, 494, 493, 525, 510, 525, 519, 531, 555, 485, 514, 515, 540, 502, 515, 524, 489, 540, 510, 495, 534, 525, 514, 516, 476, 491, 487, 487, 521, 531, 522, 532, 534, 503, 567, 467, 483, 485, 499, 479, 494, 543, 505, 493, 510, 523, 545, 516, 538, 506, 546, 506, 519, 489, 498, 512, 480] payload_rows_per_dst=[5618, 5586, 5632, 5622, 5588, 5394, 5638, 5554] dup_rows_per_dst=[2604, 2718, 2536, 2598, 2622, 2560, 2708, 2558] combine_rows_per_src=[5618, 5586, 5632, 5622, 5588, 5394, 5638, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38916 dup_rows=21116 dup_rows_remote=18480 dedup_ratio=0.322205 router_tokens_per_expert=[533, 497, 527, 515, 539, 484, 506, 519, 505, 562, 471, 486, 543, 490, 465, 497, 529, 485, 494, 497, 530, 546, 518, 460, 521, 498, 548, 527, 476, 560, 505, 546, 501, 526, 517, 512, 505, 500, 515, 514, 510, 504, 512, 520, 516, 503, 523, 522, 512, 513, 534, 494, 498, 503, 497, 519, 492, 502, 519, 486, 511, 548, 513, 548] payload_rows_per_dst=[5548, 5524, 5558, 5602, 5496, 5592, 5504, 5596] dup_rows_per_dst=[2692, 2514, 2560, 2760, 2684, 2628, 2636, 2642] combine_rows_per_src=[5548, 5524, 5558, 5602, 5496, 5592, 5504, 5596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44334 payload_rows_remote=38738 dup_rows=21202 dup_rows_remote=18542 dedup_ratio=0.323517 router_tokens_per_expert=[502, 516, 530, 518, 488, 505, 494, 502, 508, 519, 492, 544, 519, 495, 518, 503, 504, 499, 514, 470, 518, 521, 547, 530, 531, 525, 514, 509, 492, 512, 548, 497, 508, 562, 529, 494, 531, 510, 504, 485, 525, 507, 502, 533, 551, 508, 534, 503, 532, 489, 499, 477, 475, 517, 520, 479, 505, 506, 470, 527, 552, 524, 515, 511] payload_rows_per_dst=[5516, 5540, 5536, 5596, 5594, 5578, 5502, 5472] dup_rows_per_dst=[2594, 2656, 2670, 2660, 2652, 2748, 2474, 2748] combine_rows_per_src=[5516, 5540, 5536, 5596, 5594, 5578, 5502, 5472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38912 dup_rows=21010 dup_rows_remote=18420 dedup_ratio=0.320587 router_tokens_per_expert=[488, 553, 507, 476, 510, 488, 504, 487, 553, 528, 520, 530, 495, 480, 492, 515, 493, 499, 526, 469, 507, 537, 515, 507, 522, 528, 535, 531, 517, 507, 529, 524, 545, 519, 542, 482, 502, 504, 503, 505, 509, 546, 472, 516, 477, 492, 520, 515, 491, 533, 520, 540, 509, 491, 501, 491, 527, 550, 539, 517, 495, 495, 535, 513] payload_rows_per_dst=[5522, 5652, 5466, 5602, 5614, 5522, 5552, 5596] dup_rows_per_dst=[2504, 2574, 2640, 2784, 2590, 2572, 2600, 2746] combine_rows_per_src=[5522, 5652, 5466, 5602, 5614, 5522, 5552, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44154 payload_rows_remote=38686 dup_rows=21382 dup_rows_remote=18672 dedup_ratio=0.326263 router_tokens_per_expert=[484, 514, 505, 527, 498, 509, 486, 515, 560, 518, 532, 531, 515, 509, 499, 491, 506, 508, 544, 539, 512, 508, 502, 490, 500, 493, 530, 530, 523, 478, 470, 544, 530, 499, 475, 510, 501, 508, 528, 509, 509, 483, 535, 542, 541, 524, 498, 460, 501, 532, 511, 506, 536, 555, 510, 492, 553, 466, 520, 527, 509, 506, 514, 508] payload_rows_per_dst=[5468, 5566, 5652, 5414, 5556, 5594, 5588, 5316] dup_rows_per_dst=[2710, 2744, 2964, 2264, 2742, 2786, 2698, 2474] combine_rows_per_src=[5468, 5566, 5652, 5414, 5556, 5594, 5588, 5316] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38824 dup_rows=21116 dup_rows_remote=18414 dedup_ratio=0.322205 router_tokens_per_expert=[486, 499, 534, 496, 505, 516, 529, 512, 531, 497, 511, 519, 535, 557, 484, 533, 530, 514, 499, 540, 504, 550, 509, 503, 514, 502, 553, 518, 496, 530, 509, 516, 475, 519, 531, 500, 548, 531, 511, 505, 507, 489, 509, 499, 489, 477, 530, 497, 536, 508, 503, 509, 498, 496, 530, 500, 496, 522, 504, 512, 493, 532, 508, 473] payload_rows_per_dst=[5532, 5598, 5596, 5558, 5558, 5482, 5598, 5498] dup_rows_per_dst=[2622, 2736, 2702, 2718, 2682, 2512, 2562, 2582] combine_rows_per_src=[5532, 5598, 5596, 5558, 5558, 5482, 5598, 5498] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=229328 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279150544 ingress_bytes=389021696 link_bytes=389021696 token_bytes=558071760 weight_movement_bytes=110100480 total_nvlink_bytes=668172240 weight_to_token_ratio=0.197287 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765727124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277673984 dispatch_meta_bytes=229120 combine_payload_bytes=277673984 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277903104 ingress_bytes=607975424 link_bytes=607975424 token_bytes=555577088 weight_movement_bytes=330301440 total_nvlink_bytes=885878528 weight_to_token_ratio=0.594520 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057369257e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=229584 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279179472 ingress_bytes=389050368 link_bytes=389050368 token_bytes=558129360 weight_movement_bytes=110100480 total_nvlink_bytes=668229840 weight_to_token_ratio=0.197267 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766225777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278046720 dispatch_meta_bytes=229000 combine_payload_bytes=278046720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278275720 ingress_bytes=388147200 link_bytes=388147200 token_bytes=556322440 weight_movement_bytes=110100480 total_nvlink_bytes=666422920 weight_to_token_ratio=0.197908 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750518200e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=228952 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278519384 ingress_bytes=388390912 link_bytes=388390912 token_bytes=556809816 weight_movement_bytes=110100480 total_nvlink_bytes=666910296 weight_to_token_ratio=0.197734 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754756753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280369152 dispatch_meta_bytes=228952 combine_payload_bytes=280369152 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=280598104 ingress_bytes=720771072 link_bytes=720771072 token_bytes=560967256 weight_movement_bytes=440401920 total_nvlink_bytes=1001369176 weight_to_token_ratio=0.785076 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.253539441e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44154 payload_rows_remote=38686 dup_rows=21382 dup_rows_remote=18672 dedup_ratio=0.326263 router_tokens_per_expert=[484, 514, 505, 527, 498, 509, 486, 515, 560, 518, 532, 531, 515, 509, 499, 491, 506, 508, 544, 539, 512, 508, 502, 490, 500, 493, 530, 530, 523, 478, 470, 544, 530, 499, 475, 510, 501, 508, 528, 509, 509, 483, 535, 542, 541, 524, 498, 460, 501, 532, 511, 506, 536, 555, 510, 492, 553, 466, 520, 527, 509, 506, 514, 508] payload_rows_per_dst=[5468, 5566, 5652, 5414, 5556, 5594, 5588, 5316] dup_rows_per_dst=[2710, 2744, 2964, 2264, 2742, 2786, 2698, 2474] combine_rows_per_src=[5468, 5566, 5652, 5414, 5556, 5594, 5588, 5316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=228928 combine_payload_bytes=279881728 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280110656 ingress_bytes=500082688 link_bytes=500082688 token_bytes=559992384 weight_movement_bytes=220200960 total_nvlink_bytes=780193344 weight_to_token_ratio=0.393221 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697260439e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=229584 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279179472 ingress_bytes=389050368 link_bytes=389050368 token_bytes=558129360 weight_movement_bytes=110100480 total_nvlink_bytes=668229840 weight_to_token_ratio=0.197267 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766225777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277673984 dispatch_meta_bytes=229120 combine_payload_bytes=277673984 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277903104 ingress_bytes=607975424 link_bytes=607975424 token_bytes=555577088 weight_movement_bytes=330301440 total_nvlink_bytes=885878528 weight_to_token_ratio=0.594520 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057369257e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=228952 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278519384 ingress_bytes=388390912 link_bytes=388390912 token_bytes=556809816 weight_movement_bytes=110100480 total_nvlink_bytes=666910296 weight_to_token_ratio=0.197734 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754756753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=228928 combine_payload_bytes=279881728 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280110656 ingress_bytes=500082688 link_bytes=500082688 token_bytes=559992384 weight_movement_bytes=220200960 total_nvlink_bytes=780193344 weight_to_token_ratio=0.393221 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697260439e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=229328 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279150544 ingress_bytes=389021696 link_bytes=389021696 token_bytes=558071760 weight_movement_bytes=110100480 total_nvlink_bytes=668172240 weight_to_token_ratio=0.197287 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765727124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277301248 dispatch_meta_bytes=229432 combine_payload_bytes=277301248 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277530680 ingress_bytes=387401728 link_bytes=387401728 token_bytes=554831928 weight_movement_bytes=110100480 total_nvlink_bytes=664932408 weight_to_token_ratio=0.198439 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.737553216e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44424 payload_rows_remote=38846 dup_rows=21112 dup_rows_remote=18556 dedup_ratio=0.322144 router_tokens_per_expert=[495, 521, 527, 504, 518, 483, 498, 537, 479, 539, 569, 520, 521, 547, 511, 496, 491, 485, 534, 534, 521, 523, 508, 535, 533, 511, 501, 495, 491, 509, 529, 510, 473, 511, 531, 498, 510, 495, 540, 509, 509, 486, 514, 481, 532, 512, 481, 531, 505, 522, 522, 522, 505, 491, 526, 507, 512, 529, 517, 480, 506, 521, 489, 526] payload_rows_per_dst=[5558, 5606, 5636, 5502, 5578, 5500, 5518, 5526] dup_rows_per_dst=[2608, 2758, 2626, 2656, 2556, 2592, 2682, 2634] combine_rows_per_src=[5558, 5606, 5636, 5502, 5578, 5500, 5518, 5526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38830 dup_rows=21094 dup_rows_remote=18446 dedup_ratio=0.321869 router_tokens_per_expert=[535, 490, 511, 490, 535, 523, 489, 471, 505, 552, 505, 517, 517, 480, 517, 511, 480, 523, 507, 525, 493, 534, 516, 493, 513, 483, 533, 539, 551, 533, 538, 462, 520, 526, 495, 467, 536, 519, 534, 531, 532, 546, 509, 540, 506, 483, 488, 526, 492, 516, 508, 483, 560, 528, 520, 465, 520, 514, 504, 508, 519, 525, 489, 488] payload_rows_per_dst=[5482, 5610, 5522, 5602, 5554, 5612, 5570, 5490] dup_rows_per_dst=[2606, 2598, 2620, 2702, 2702, 2648, 2574, 2644] combine_rows_per_src=[5482, 5610, 5522, 5602, 5554, 5612, 5570, 5490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=38988 dup_rows=21024 dup_rows_remote=18404 dedup_ratio=0.320801 router_tokens_per_expert=[526, 495, 539, 541, 510, 513, 537, 478, 475, 530, 465, 510, 511, 542, 525, 514, 500, 495, 503, 515, 493, 519, 504, 517, 540, 524, 466, 511, 524, 505, 533, 506, 536, 526, 498, 519, 510, 519, 542, 509, 509, 497, 476, 520, 512, 546, 495, 480, 475, 484, 552, 500, 489, 511, 513, 517, 489, 531, 487, 544, 499, 575, 523, 519] payload_rows_per_dst=[5546, 5524, 5532, 5574, 5702, 5530, 5492, 5612] dup_rows_per_dst=[2732, 2620, 2560, 2644, 2616, 2540, 2590, 2722] combine_rows_per_src=[5546, 5524, 5532, 5574, 5702, 5530, 5492, 5612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38742 dup_rows=21100 dup_rows_remote=18234 dedup_ratio=0.321960 router_tokens_per_expert=[472, 505, 518, 549, 542, 512, 502, 509, 495, 552, 540, 526, 510, 510, 535, 521, 477, 480, 506, 504, 515, 509, 557, 523, 493, 507, 486, 521, 513, 502, 527, 508, 513, 458, 509, 527, 494, 558, 504, 510, 509, 531, 531, 523, 472, 522, 532, 519, 486, 512, 534, 525, 500, 510, 499, 507, 535, 503, 490, 528, 521, 495, 490, 495] payload_rows_per_dst=[5628, 5420, 5512, 5542, 5466, 5644, 5530, 5694] dup_rows_per_dst=[2590, 2512, 2630, 2512, 2548, 2820, 2622, 2866] combine_rows_per_src=[5628, 5420, 5512, 5542, 5466, 5644, 5530, 5694] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38850 dup_rows=21134 dup_rows_remote=18598 dedup_ratio=0.322479 router_tokens_per_expert=[533, 553, 504, 513, 491, 500, 504, 521, 532, 541, 497, 544, 534, 504, 511, 498, 493, 508, 531, 524, 503, 548, 520, 495, 509, 479, 569, 493, 504, 539, 492, 487, 510, 493, 519, 534, 548, 486, 539, 557, 488, 506, 495, 553, 467, 505, 490, 518, 495, 545, 502, 504, 532, 478, 537, 451, 533, 490, 519, 468, 484, 536, 498, 514] payload_rows_per_dst=[5568, 5570, 5566, 5544, 5654, 5496, 5552, 5452] dup_rows_per_dst=[2670, 2752, 2678, 2600, 2718, 2548, 2536, 2632] combine_rows_per_src=[5568, 5570, 5566, 5544, 5654, 5496, 5552, 5452] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44540 payload_rows_remote=38932 dup_rows=20996 dup_rows_remote=18286 dedup_ratio=0.320374 router_tokens_per_expert=[486, 521, 504, 498, 535, 471, 483, 523, 469, 553, 485, 525, 568, 521, 499, 523, 548, 489, 493, 518, 509, 485, 484, 475, 474, 548, 509, 510, 549, 539, 507, 523, 536, 516, 507, 528, 523, 535, 537, 504, 500, 516, 494, 496, 498, 524, 525, 512, 500, 488, 526, 546, 546, 509, 536, 517, 511, 496, 506, 496, 536, 489, 505, 486] payload_rows_per_dst=[5512, 5602, 5510, 5608, 5666, 5542, 5626, 5474] dup_rows_per_dst=[2530, 2684, 2492, 2710, 2706, 2588, 2710, 2576] combine_rows_per_src=[5512, 5602, 5510, 5608, 5666, 5542, 5626, 5474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38830 dup_rows=21094 dup_rows_remote=18446 dedup_ratio=0.321869 router_tokens_per_expert=[535, 490, 511, 490, 535, 523, 489, 471, 505, 552, 505, 517, 517, 480, 517, 511, 480, 523, 507, 525, 493, 534, 516, 493, 513, 483, 533, 539, 551, 533, 538, 462, 520, 526, 495, 467, 536, 519, 534, 531, 532, 546, 509, 540, 506, 483, 488, 526, 492, 516, 508, 483, 560, 528, 520, 465, 520, 514, 504, 508, 519, 525, 489, 488] payload_rows_per_dst=[5482, 5610, 5522, 5602, 5554, 5612, 5570, 5490] dup_rows_per_dst=[2606, 2598, 2620, 2702, 2702, 2648, 2574, 2644] combine_rows_per_src=[5482, 5610, 5522, 5602, 5554, 5612, 5570, 5490] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38742 dup_rows=21100 dup_rows_remote=18234 dedup_ratio=0.321960 router_tokens_per_expert=[472, 505, 518, 549, 542, 512, 502, 509, 495, 552, 540, 526, 510, 510, 535, 521, 477, 480, 506, 504, 515, 509, 557, 523, 493, 507, 486, 521, 513, 502, 527, 508, 513, 458, 509, 527, 494, 558, 504, 510, 509, 531, 531, 523, 472, 522, 532, 519, 486, 512, 534, 525, 500, 510, 499, 507, 535, 503, 490, 528, 521, 495, 490, 495] payload_rows_per_dst=[5628, 5420, 5512, 5542, 5466, 5644, 5530, 5694] dup_rows_per_dst=[2590, 2512, 2630, 2512, 2548, 2820, 2622, 2866] combine_rows_per_src=[5628, 5420, 5512, 5542, 5466, 5644, 5530, 5694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38976 dup_rows=21000 dup_rows_remote=18368 dedup_ratio=0.320435 router_tokens_per_expert=[492, 495, 524, 435, 506, 507, 482, 517, 515, 505, 494, 486, 514, 506, 539, 533, 513, 522, 515, 523, 504, 501, 485, 533, 526, 541, 523, 529, 529, 487, 530, 450, 541, 505, 525, 509, 532, 480, 524, 502, 493, 493, 539, 521, 481, 520, 516, 502, 528, 539, 537, 512, 535, 497, 502, 550, 537, 486, 497, 508, 523, 523, 517, 533] payload_rows_per_dst=[5478, 5568, 5560, 5592, 5602, 5438, 5726, 5572] dup_rows_per_dst=[2438, 2616, 2632, 2638, 2634, 2692, 2674, 2676] combine_rows_per_src=[5478, 5568, 5560, 5592, 5602, 5438, 5726, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=38988 dup_rows=21024 dup_rows_remote=18404 dedup_ratio=0.320801 router_tokens_per_expert=[526, 495, 539, 541, 510, 513, 537, 478, 475, 530, 465, 510, 511, 542, 525, 514, 500, 495, 503, 515, 493, 519, 504, 517, 540, 524, 466, 511, 524, 505, 533, 506, 536, 526, 498, 519, 510, 519, 542, 509, 509, 497, 476, 520, 512, 546, 495, 480, 475, 484, 552, 500, 489, 511, 513, 517, 489, 531, 487, 544, 499, 575, 523, 519] payload_rows_per_dst=[5546, 5524, 5532, 5574, 5702, 5530, 5492, 5612] dup_rows_per_dst=[2732, 2620, 2560, 2644, 2616, 2540, 2590, 2722] combine_rows_per_src=[5546, 5524, 5532, 5574, 5702, 5530, 5492, 5612] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44424 payload_rows_remote=38846 dup_rows=21112 dup_rows_remote=18556 dedup_ratio=0.322144 router_tokens_per_expert=[495, 521, 527, 504, 518, 483, 498, 537, 479, 539, 569, 520, 521, 547, 511, 496, 491, 485, 534, 534, 521, 523, 508, 535, 533, 511, 501, 495, 491, 509, 529, 510, 473, 511, 531, 498, 510, 495, 540, 509, 509, 486, 514, 481, 532, 512, 481, 531, 505, 522, 522, 522, 505, 491, 526, 507, 512, 529, 517, 480, 506, 521, 489, 526] payload_rows_per_dst=[5558, 5606, 5636, 5502, 5578, 5500, 5518, 5526] dup_rows_per_dst=[2608, 2758, 2626, 2656, 2556, 2592, 2682, 2634] combine_rows_per_src=[5558, 5606, 5636, 5502, 5578, 5500, 5518, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=229792 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278706592 ingress_bytes=388577280 link_bytes=388577280 token_bytes=557183392 weight_movement_bytes=110100480 total_nvlink_bytes=667283872 weight_to_token_ratio=0.197602 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757997999e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44716 payload_rows_remote=38928 dup_rows=20820 dup_rows_remote=18018 dedup_ratio=0.317688 router_tokens_per_expert=[515, 523, 555, 548, 545, 535, 490, 491, 508, 462, 499, 524, 528, 509, 558, 529, 506, 559, 508, 498, 487, 522, 477, 521, 506, 482, 520, 541, 495, 501, 526, 508, 501, 521, 497, 496, 545, 496, 500, 513, 572, 514, 544, 529, 501, 498, 503, 512, 486, 542, 513, 512, 531, 489, 523, 471, 505, 502, 507, 524, 472, 479, 504, 490] payload_rows_per_dst=[5788, 5576, 5710, 5568, 5404, 5638, 5522, 5510] dup_rows_per_dst=[2802, 2520, 2750, 2542, 2430, 2708, 2612, 2456] combine_rows_per_src=[5788, 5576, 5710, 5568, 5404, 5638, 5522, 5510] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278448128 dispatch_meta_bytes=229608 combine_payload_bytes=278448128 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278677736 ingress_bytes=608749568 link_bytes=608749568 token_bytes=557125864 weight_movement_bytes=330301440 total_nvlink_bytes=887427304 weight_to_token_ratio=0.592867 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058715621e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38976 dup_rows=21000 dup_rows_remote=18368 dedup_ratio=0.320435 router_tokens_per_expert=[492, 495, 524, 435, 506, 507, 482, 517, 515, 505, 494, 486, 514, 506, 539, 533, 513, 522, 515, 523, 504, 501, 485, 533, 526, 541, 523, 529, 529, 487, 530, 450, 541, 505, 525, 509, 532, 480, 524, 502, 493, 493, 539, 521, 481, 520, 516, 502, 528, 539, 537, 512, 535, 497, 502, 550, 537, 486, 497, 508, 523, 523, 517, 533] payload_rows_per_dst=[5478, 5568, 5560, 5592, 5602, 5438, 5726, 5572] dup_rows_per_dst=[2438, 2616, 2632, 2638, 2634, 2692, 2674, 2676] combine_rows_per_src=[5478, 5568, 5560, 5592, 5602, 5438, 5726, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=229104 combine_payload_bytes=278333440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278562544 ingress_bytes=388433920 link_bytes=388433920 token_bytes=556895984 weight_movement_bytes=110100480 total_nvlink_bytes=666996464 weight_to_token_ratio=0.197704 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755504733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=229568 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279695552 ingress_bytes=609767424 link_bytes=609767424 token_bytes=559161536 weight_movement_bytes=330301440 total_nvlink_bytes=889462976 weight_to_token_ratio=0.590708 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060485840e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277702656 dispatch_meta_bytes=227904 combine_payload_bytes=277702656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277930560 ingress_bytes=387803136 link_bytes=387803136 token_bytes=555633216 weight_movement_bytes=110100480 total_nvlink_bytes=665733696 weight_to_token_ratio=0.198153 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744534362e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=229792 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278706592 ingress_bytes=388577280 link_bytes=388577280 token_bytes=557183392 weight_movement_bytes=110100480 total_nvlink_bytes=667283872 weight_to_token_ratio=0.197602 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757997999e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=228872 combine_payload_bytes=279064576 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279293448 ingress_bytes=609366016 link_bytes=609366016 token_bytes=558358024 weight_movement_bytes=330301440 total_nvlink_bytes=888659464 weight_to_token_ratio=0.591559 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059787725e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277702656 dispatch_meta_bytes=227904 combine_payload_bytes=277702656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277930560 ingress_bytes=387803136 link_bytes=387803136 token_bytes=555633216 weight_movement_bytes=110100480 total_nvlink_bytes=665733696 weight_to_token_ratio=0.198153 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744534362e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279035904 dispatch_meta_bytes=227784 combine_payload_bytes=279035904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279263688 ingress_bytes=389136384 link_bytes=389136384 token_bytes=558299592 weight_movement_bytes=110100480 total_nvlink_bytes=668400072 weight_to_token_ratio=0.197207 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767721737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=229376 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279609344 ingress_bytes=389480448 link_bytes=389480448 token_bytes=558989312 weight_movement_bytes=110100480 total_nvlink_bytes=669089792 weight_to_token_ratio=0.196963 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773705576e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44638 payload_rows_remote=38972 dup_rows=20898 dup_rows_remote=18212 dedup_ratio=0.318878 router_tokens_per_expert=[496, 505, 521, 525, 518, 505, 482, 552, 509, 539, 509, 543, 515, 508, 512, 523, 541, 527, 543, 516, 495, 496, 469, 480, 512, 485, 540, 513, 512, 501, 516, 487, 544, 527, 478, 508, 525, 528, 547, 519, 513, 519, 454, 530, 517, 501, 475, 505, 504, 501, 509, 510, 513, 512, 511, 478, 501, 525, 531, 497, 471, 550, 540, 530] payload_rows_per_dst=[5624, 5602, 5560, 5538, 5666, 5458, 5508, 5682] dup_rows_per_dst=[2584, 2714, 2574, 2594, 2686, 2570, 2568, 2608] combine_rows_per_src=[5624, 5602, 5560, 5538, 5666, 5458, 5508, 5682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=229568 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279695552 ingress_bytes=609767424 link_bytes=609767424 token_bytes=559161536 weight_movement_bytes=330301440 total_nvlink_bytes=889462976 weight_to_token_ratio=0.590708 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060485840e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278448128 dispatch_meta_bytes=229608 combine_payload_bytes=278448128 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278677736 ingress_bytes=608749568 link_bytes=608749568 token_bytes=557125864 weight_movement_bytes=330301440 total_nvlink_bytes=887427304 weight_to_token_ratio=0.592867 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058715621e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279035904 dispatch_meta_bytes=227784 combine_payload_bytes=279035904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279263688 ingress_bytes=389136384 link_bytes=389136384 token_bytes=558299592 weight_movement_bytes=110100480 total_nvlink_bytes=668400072 weight_to_token_ratio=0.197207 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767721737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44556 payload_rows_remote=39018 dup_rows=20980 dup_rows_remote=18386 dedup_ratio=0.320129 router_tokens_per_expert=[471, 518, 519, 510, 512, 548, 512, 482, 529, 502, 512, 525, 518, 471, 515, 491, 517, 505, 528, 521, 539, 499, 527, 563, 553, 499, 519, 509, 527, 484, 509, 468, 487, 503, 509, 487, 479, 516, 503, 541, 519, 517, 533, 515, 488, 537, 485, 504, 496, 517, 513, 513, 500, 521, 493, 513, 506, 520, 489, 526, 535, 533, 505, 563] payload_rows_per_dst=[5496, 5530, 5630, 5564, 5532, 5626, 5538, 5640] dup_rows_per_dst=[2648, 2596, 2768, 2572, 2518, 2570, 2594, 2714] combine_rows_per_src=[5496, 5530, 5630, 5564, 5532, 5626, 5538, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44584 payload_rows_remote=38996 dup_rows=20952 dup_rows_remote=18452 dedup_ratio=0.319702 router_tokens_per_expert=[505, 493, 558, 500, 528, 521, 493, 518, 525, 483, 522, 545, 520, 515, 503, 526, 531, 499, 568, 502, 518, 508, 551, 541, 484, 507, 498, 497, 535, 479, 531, 513, 480, 533, 503, 497, 511, 530, 504, 506, 485, 485, 538, 514, 520, 534, 493, 484, 504, 546, 532, 497, 516, 531, 512, 499, 523, 488, 499, 469, 524, 492, 474, 528] payload_rows_per_dst=[5632, 5634, 5684, 5588, 5502, 5466, 5618, 5460] dup_rows_per_dst=[2600, 2644, 2752, 2500, 2626, 2640, 2656, 2534] combine_rows_per_src=[5632, 5634, 5684, 5588, 5502, 5466, 5618, 5460] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=39032 dup_rows=21004 dup_rows_remote=18448 dedup_ratio=0.320496 router_tokens_per_expert=[476, 536, 512, 492, 564, 532, 545, 488, 531, 527, 501, 497, 515, 516, 554, 481, 538, 534, 566, 493, 521, 508, 523, 535, 525, 483, 525, 520, 517, 500, 445, 469, 483, 498, 472, 501, 525, 537, 494, 516, 509, 527, 472, 497, 495, 519, 512, 497, 500, 533, 518, 541, 482, 497, 493, 486, 520, 522, 537, 547, 534, 493, 501, 541] payload_rows_per_dst=[5594, 5630, 5678, 5470, 5510, 5500, 5520, 5630] dup_rows_per_dst=[2696, 2614, 2758, 2498, 2542, 2556, 2580, 2760] combine_rows_per_src=[5594, 5630, 5678, 5470, 5510, 5500, 5520, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44238 payload_rows_remote=38732 dup_rows=21298 dup_rows_remote=18676 dedup_ratio=0.324982 router_tokens_per_expert=[550, 526, 502, 479, 508, 492, 517, 505, 491, 500, 498, 485, 529, 510, 498, 480, 512, 479, 540, 509, 495, 526, 522, 552, 511, 531, 541, 490, 504, 518, 527, 507, 515, 529, 513, 523, 517, 522, 527, 544, 525, 524, 504, 488, 534, 543, 518, 506, 498, 509, 527, 491, 492, 523, 511, 492, 546, 510, 481, 487, 547, 525, 483, 480] payload_rows_per_dst=[5270, 5304, 5278, 5720, 5638, 5766, 5756, 5506] dup_rows_per_dst=[2370, 2262, 2342, 3056, 2742, 2934, 2970, 2622] combine_rows_per_src=[5270, 5304, 5278, 5720, 5638, 5766, 5756, 5506] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38858 dup_rows=21088 dup_rows_remote=18430 dedup_ratio=0.321777 router_tokens_per_expert=[531, 500, 542, 530, 547, 522, 513, 488, 506, 532, 523, 477, 549, 543, 497, 497, 519, 545, 532, 536, 494, 464, 525, 469, 529, 487, 488, 533, 510, 522, 525, 538, 462, 522, 520, 481, 519, 539, 506, 495, 510, 507, 505, 502, 528, 546, 528, 488, 471, 528, 537, 478, 529, 491, 508, 491, 493, 475, 541, 514, 541, 531, 497, 472] payload_rows_per_dst=[5588, 5590, 5546, 5542, 5588, 5544, 5534, 5516] dup_rows_per_dst=[2758, 2658, 2622, 2722, 2500, 2684, 2532, 2612] combine_rows_per_src=[5588, 5590, 5546, 5542, 5588, 5544, 5534, 5516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44556 payload_rows_remote=39018 dup_rows=20980 dup_rows_remote=18386 dedup_ratio=0.320129 router_tokens_per_expert=[471, 518, 519, 510, 512, 548, 512, 482, 529, 502, 512, 525, 518, 471, 515, 491, 517, 505, 528, 521, 539, 499, 527, 563, 553, 499, 519, 509, 527, 484, 509, 468, 487, 503, 509, 487, 479, 516, 503, 541, 519, 517, 533, 515, 488, 537, 485, 504, 496, 517, 513, 513, 500, 521, 493, 513, 506, 520, 489, 526, 535, 533, 505, 563] payload_rows_per_dst=[5496, 5530, 5630, 5564, 5532, 5626, 5538, 5640] dup_rows_per_dst=[2648, 2596, 2768, 2572, 2518, 2570, 2594, 2714] combine_rows_per_src=[5496, 5530, 5630, 5564, 5532, 5626, 5538, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229616 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279910640 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591664 weight_movement_bytes=110100480 total_nvlink_bytes=669692144 weight_to_token_ratio=0.196751 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44238 payload_rows_remote=38732 dup_rows=21298 dup_rows_remote=18676 dedup_ratio=0.324982 router_tokens_per_expert=[550, 526, 502, 479, 508, 492, 517, 505, 491, 500, 498, 485, 529, 510, 498, 480, 512, 479, 540, 509, 495, 526, 522, 552, 511, 531, 541, 490, 504, 518, 527, 507, 515, 529, 513, 523, 517, 522, 527, 544, 525, 524, 504, 488, 534, 543, 518, 506, 498, 509, 527, 491, 492, 523, 511, 492, 546, 510, 481, 487, 547, 525, 483, 480] payload_rows_per_dst=[5270, 5304, 5278, 5720, 5638, 5766, 5756, 5506] dup_rows_per_dst=[2370, 2262, 2342, 3056, 2742, 2934, 2970, 2622] combine_rows_per_src=[5270, 5304, 5278, 5720, 5638, 5766, 5756, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44584 payload_rows_remote=38996 dup_rows=20952 dup_rows_remote=18452 dedup_ratio=0.319702 router_tokens_per_expert=[505, 493, 558, 500, 528, 521, 493, 518, 525, 483, 522, 545, 520, 515, 503, 526, 531, 499, 568, 502, 518, 508, 551, 541, 484, 507, 498, 497, 535, 479, 531, 513, 480, 533, 503, 497, 511, 530, 504, 506, 485, 485, 538, 514, 520, 534, 493, 484, 504, 546, 532, 497, 516, 531, 512, 499, 523, 488, 499, 469, 524, 492, 474, 528] payload_rows_per_dst=[5632, 5634, 5684, 5588, 5502, 5466, 5618, 5460] dup_rows_per_dst=[2600, 2644, 2752, 2500, 2626, 2640, 2656, 2534] combine_rows_per_src=[5632, 5634, 5684, 5588, 5502, 5466, 5618, 5460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=228736 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279580032 ingress_bytes=389451776 link_bytes=389451776 token_bytes=558931328 weight_movement_bytes=110100480 total_nvlink_bytes=669031808 weight_to_token_ratio=0.196984 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773206923e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44270 payload_rows_remote=38750 dup_rows=21266 dup_rows_remote=18558 dedup_ratio=0.324493 router_tokens_per_expert=[544, 525, 486, 539, 506, 549, 496, 527, 503, 533, 519, 527, 515, 505, 511, 515, 537, 514, 493, 517, 522, 500, 556, 475, 486, 492, 526, 477, 509, 492, 521, 545, 521, 527, 523, 520, 495, 533, 516, 505, 523, 496, 470, 516, 519, 517, 503, 491, 493, 497, 516, 487, 476, 558, 528, 506, 501, 481, 519, 529, 491, 503, 526, 520] payload_rows_per_dst=[5584, 5566, 5520, 5480, 5562, 5504, 5504, 5550] dup_rows_per_dst=[2760, 2690, 2708, 2616, 2718, 2566, 2618, 2590] combine_rows_per_src=[5584, 5566, 5520, 5480, 5562, 5504, 5504, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44638 payload_rows_remote=38972 dup_rows=20898 dup_rows_remote=18212 dedup_ratio=0.318878 router_tokens_per_expert=[496, 505, 521, 525, 518, 505, 482, 552, 509, 539, 509, 543, 515, 508, 512, 523, 541, 527, 543, 516, 495, 496, 469, 480, 512, 485, 540, 513, 512, 501, 516, 487, 544, 527, 478, 508, 525, 528, 547, 519, 513, 519, 454, 530, 517, 501, 475, 505, 504, 501, 509, 510, 513, 512, 511, 478, 501, 525, 531, 497, 471, 550, 540, 530] payload_rows_per_dst=[5624, 5602, 5560, 5538, 5666, 5458, 5508, 5682] dup_rows_per_dst=[2584, 2714, 2574, 2594, 2686, 2570, 2568, 2608] combine_rows_per_src=[5624, 5602, 5560, 5538, 5666, 5458, 5508, 5682] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38908 dup_rows=21042 dup_rows_remote=18296 dedup_ratio=0.321075 router_tokens_per_expert=[505, 536, 514, 523, 515, 514, 470, 526, 569, 527, 483, 512, 525, 532, 521, 519, 508, 509, 495, 523, 517, 488, 521, 560, 517, 507, 526, 502, 475, 490, 483, 538, 563, 494, 504, 522, 490, 525, 487, 495, 545, 487, 529, 480, 504, 504, 520, 507, 489, 482, 517, 515, 483, 536, 491, 507, 507, 515, 555, 537, 507, 523, 506, 492] payload_rows_per_dst=[5586, 5700, 5622, 5492, 5572, 5468, 5468, 5586] dup_rows_per_dst=[2746, 2802, 2494, 2584, 2634, 2684, 2572, 2526] combine_rows_per_src=[5586, 5700, 5622, 5492, 5572, 5468, 5468, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279781376 dispatch_meta_bytes=229920 combine_payload_bytes=279781376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280011296 ingress_bytes=389881856 link_bytes=389881856 token_bytes=559792672 weight_movement_bytes=110100480 total_nvlink_bytes=669893152 weight_to_token_ratio=0.196681 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780686721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=229152 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278763296 ingress_bytes=498735104 link_bytes=498735104 token_bytes=557297440 weight_movement_bytes=220200960 total_nvlink_bytes=777498400 weight_to_token_ratio=0.395123 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673823737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44270 payload_rows_remote=38750 dup_rows=21266 dup_rows_remote=18558 dedup_ratio=0.324493 router_tokens_per_expert=[544, 525, 486, 539, 506, 549, 496, 527, 503, 533, 519, 527, 515, 505, 511, 515, 537, 514, 493, 517, 522, 500, 556, 475, 486, 492, 526, 477, 509, 492, 521, 545, 521, 527, 523, 520, 495, 533, 516, 505, 523, 496, 470, 516, 519, 517, 503, 491, 493, 497, 516, 487, 476, 558, 528, 506, 501, 481, 519, 529, 491, 503, 526, 520] payload_rows_per_dst=[5584, 5566, 5520, 5480, 5562, 5504, 5504, 5550] dup_rows_per_dst=[2760, 2690, 2708, 2616, 2718, 2566, 2618, 2590] combine_rows_per_src=[5584, 5566, 5520, 5480, 5562, 5504, 5504, 5550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229616 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279910640 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591664 weight_movement_bytes=110100480 total_nvlink_bytes=669692144 weight_to_token_ratio=0.196751 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277630976 dispatch_meta_bytes=229632 combine_payload_bytes=277630976 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277860608 ingress_bytes=607932416 link_bytes=607932416 token_bytes=555491584 weight_movement_bytes=330301440 total_nvlink_bytes=885793024 weight_to_token_ratio=0.594611 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057294459e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=229792 combine_payload_bytes=279523328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279753120 ingress_bytes=389623808 link_bytes=389623808 token_bytes=559276448 weight_movement_bytes=110100480 total_nvlink_bytes=669376928 weight_to_token_ratio=0.196862 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776198842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278892544 dispatch_meta_bytes=228816 combine_payload_bytes=278892544 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279121360 ingress_bytes=499093504 link_bytes=499093504 token_bytes=558013904 weight_movement_bytes=220200960 total_nvlink_bytes=778214864 weight_to_token_ratio=0.394616 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680056902e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=229792 combine_payload_bytes=279523328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279753120 ingress_bytes=389623808 link_bytes=389623808 token_bytes=559276448 weight_movement_bytes=110100480 total_nvlink_bytes=669376928 weight_to_token_ratio=0.196862 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776198842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277630976 dispatch_meta_bytes=229632 combine_payload_bytes=277630976 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277860608 ingress_bytes=607932416 link_bytes=607932416 token_bytes=555491584 weight_movement_bytes=330301440 total_nvlink_bytes=885793024 weight_to_token_ratio=0.594611 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057294459e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=229152 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278763296 ingress_bytes=498735104 link_bytes=498735104 token_bytes=557297440 weight_movement_bytes=220200960 total_nvlink_bytes=777498400 weight_to_token_ratio=0.395123 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673823737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277760000 dispatch_meta_bytes=229232 combine_payload_bytes=277760000 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=277989232 ingress_bytes=608061440 link_bytes=608061440 token_bytes=555749232 weight_movement_bytes=330301440 total_nvlink_bytes=886050672 weight_to_token_ratio=0.594335 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057518853e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44280 payload_rows_remote=38628 dup_rows=21256 dup_rows_remote=18456 dedup_ratio=0.324341 router_tokens_per_expert=[511, 500, 533, 535, 517, 530, 485, 521, 489, 504, 488, 534, 514, 482, 468, 512, 511, 517, 531, 523, 521, 541, 527, 507, 461, 505, 516, 547, 555, 508, 537, 499, 529, 532, 543, 546, 539, 508, 517, 512, 485, 511, 507, 472, 559, 459, 523, 483, 522, 485, 523, 521, 503, 495, 508, 509, 537, 474, 496, 492, 517, 480, 522, 530] payload_rows_per_dst=[5588, 5428, 5562, 5560, 5652, 5506, 5524, 5460] dup_rows_per_dst=[2676, 2554, 2794, 2696, 2800, 2492, 2608, 2636] combine_rows_per_src=[5588, 5428, 5562, 5560, 5652, 5506, 5524, 5460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44296 payload_rows_remote=38674 dup_rows=21240 dup_rows_remote=18568 dedup_ratio=0.324097 router_tokens_per_expert=[551, 526, 479, 487, 520, 495, 496, 549, 516, 509, 516, 514, 491, 525, 530, 502, 571, 494, 495, 506, 510, 489, 510, 537, 509, 523, 491, 469, 513, 533, 526, 513, 508, 552, 506, 467, 534, 529, 529, 478, 497, 527, 517, 526, 548, 518, 499, 515, 537, 523, 501, 480, 488, 484, 541, 534, 514, 516, 491, 468, 529, 492, 497, 528] payload_rows_per_dst=[5584, 5498, 5544, 5456, 5576, 5622, 5560, 5456] dup_rows_per_dst=[2622, 2708, 2680, 2698, 2630, 2672, 2616, 2614] combine_rows_per_src=[5584, 5498, 5544, 5456, 5576, 5622, 5560, 5456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=39018 dup_rows=20978 dup_rows_remote=18396 dedup_ratio=0.320099 router_tokens_per_expert=[530, 503, 498, 534, 532, 523, 518, 509, 544, 505, 499, 509, 517, 516, 498, 473, 536, 486, 540, 493, 482, 501, 512, 502, 517, 509, 517, 552, 520, 480, 509, 505, 524, 507, 495, 529, 486, 512, 466, 495, 555, 500, 541, 508, 490, 529, 503, 501, 511, 502, 477, 493, 520, 533, 513, 514, 491, 495, 543, 534, 543, 549, 518, 522] payload_rows_per_dst=[5526, 5540, 5614, 5606, 5534, 5584, 5484, 5670] dup_rows_per_dst=[2768, 2582, 2490, 2612, 2494, 2670, 2642, 2720] combine_rows_per_src=[5526, 5540, 5614, 5606, 5534, 5584, 5484, 5670] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=228736 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279580032 ingress_bytes=389451776 link_bytes=389451776 token_bytes=558931328 weight_movement_bytes=110100480 total_nvlink_bytes=669031808 weight_to_token_ratio=0.196984 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773206923e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278892544 dispatch_meta_bytes=228816 combine_payload_bytes=278892544 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279121360 ingress_bytes=499093504 link_bytes=499093504 token_bytes=558013904 weight_movement_bytes=220200960 total_nvlink_bytes=778214864 weight_to_token_ratio=0.394616 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680056902e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38972 dup_rows=20926 dup_rows_remote=18260 dedup_ratio=0.319305 router_tokens_per_expert=[494, 498, 478, 499, 467, 501, 459, 519, 527, 523, 548, 477, 540, 526, 516, 507, 510, 539, 532, 527, 525, 484, 520, 502, 481, 542, 476, 494, 485, 516, 536, 522, 509, 495, 527, 508, 513, 481, 527, 484, 524, 522, 566, 527, 493, 522, 512, 522, 527, 529, 531, 489, 529, 578, 488, 481, 536, 543, 501, 521, 513, 517, 507, 476] payload_rows_per_dst=[5408, 5628, 5592, 5554, 5504, 5674, 5638, 5612] dup_rows_per_dst=[2422, 2700, 2686, 2550, 2584, 2702, 2666, 2616] combine_rows_per_src=[5408, 5628, 5592, 5554, 5504, 5674, 5638, 5612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276885504 dispatch_meta_bytes=228336 combine_payload_bytes=276885504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277113840 ingress_bytes=386985984 link_bytes=386985984 token_bytes=553999344 weight_movement_bytes=110100480 total_nvlink_bytes=664099824 weight_to_token_ratio=0.198738 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.730322744e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44296 payload_rows_remote=38674 dup_rows=21240 dup_rows_remote=18568 dedup_ratio=0.324097 router_tokens_per_expert=[551, 526, 479, 487, 520, 495, 496, 549, 516, 509, 516, 514, 491, 525, 530, 502, 571, 494, 495, 506, 510, 489, 510, 537, 509, 523, 491, 469, 513, 533, 526, 513, 508, 552, 506, 467, 534, 529, 529, 478, 497, 527, 517, 526, 548, 518, 499, 515, 537, 523, 501, 480, 488, 484, 541, 534, 514, 516, 491, 468, 529, 492, 497, 528] payload_rows_per_dst=[5584, 5498, 5544, 5456, 5576, 5622, 5560, 5456] dup_rows_per_dst=[2622, 2708, 2680, 2698, 2630, 2672, 2616, 2614] combine_rows_per_src=[5584, 5498, 5544, 5456, 5576, 5622, 5560, 5456] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=38828 dup_rows=21110 dup_rows_remote=18394 dedup_ratio=0.322113 router_tokens_per_expert=[497, 482, 533, 515, 500, 514, 525, 538, 576, 500, 511, 512, 481, 520, 462, 521, 480, 504, 549, 511, 500, 523, 519, 525, 534, 487, 539, 502, 525, 519, 526, 525, 527, 532, 542, 547, 461, 526, 539, 466, 488, 517, 545, 530, 478, 515, 476, 446, 490, 515, 536, 496, 536, 529, 489, 500, 521, 501, 537, 515, 512, 509, 485, 507] payload_rows_per_dst=[5560, 5590, 5558, 5598, 5626, 5422, 5592, 5480] dup_rows_per_dst=[2648, 2576, 2664, 2716, 2654, 2568, 2590, 2694] combine_rows_per_src=[5560, 5590, 5558, 5598, 5626, 5422, 5592, 5480] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38740 dup_rows=21146 dup_rows_remote=18422 dedup_ratio=0.322662 router_tokens_per_expert=[521, 503, 529, 480, 500, 467, 505, 505, 500, 529, 531, 515, 488, 530, 543, 499, 499, 513, 524, 506, 527, 517, 492, 530, 554, 545, 507, 498, 498, 549, 520, 523, 544, 454, 477, 506, 487, 504, 517, 481, 518, 508, 504, 521, 524, 530, 470, 519, 501, 495, 492, 527, 513, 491, 496, 555, 541, 520, 529, 491, 527, 561, 491, 527] payload_rows_per_dst=[5490, 5572, 5538, 5570, 5386, 5616, 5568, 5650] dup_rows_per_dst=[2530, 2698, 2678, 2676, 2554, 2624, 2662, 2724] combine_rows_per_src=[5490, 5572, 5538, 5570, 5386, 5616, 5568, 5650] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229656 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279910680 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591704 weight_movement_bytes=110100480 total_nvlink_bytes=669692184 weight_to_token_ratio=0.196751 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277215232 dispatch_meta_bytes=228968 combine_payload_bytes=277215232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277444200 ingress_bytes=387315712 link_bytes=387315712 token_bytes=554659432 weight_movement_bytes=110100480 total_nvlink_bytes=664759912 weight_to_token_ratio=0.198501 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.736057257e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38620 dup_rows=21126 dup_rows_remote=18230 dedup_ratio=0.322357 router_tokens_per_expert=[484, 494, 499, 542, 508, 495, 528, 527, 511, 485, 515, 515, 521, 495, 535, 484, 526, 545, 543, 522, 506, 517, 533, 501, 518, 518, 516, 501, 514, 535, 532, 511, 505, 512, 519, 515, 512, 520, 527, 522, 517, 518, 517, 534, 483, 525, 459, 490, 523, 515, 490, 503, 536, 501, 509, 484, 503, 519, 509, 437, 487, 529, 550, 522] payload_rows_per_dst=[5790, 5510, 5514, 5528, 5578, 5532, 5500, 5458] dup_rows_per_dst=[2896, 2768, 2542, 2576, 2696, 2554, 2622, 2472] combine_rows_per_src=[5790, 5510, 5514, 5528, 5578, 5532, 5500, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=228928 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279580224 ingress_bytes=389451776 link_bytes=389451776 token_bytes=558931520 weight_movement_bytes=110100480 total_nvlink_bytes=669032000 weight_to_token_ratio=0.196984 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773206923e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=39018 dup_rows=20978 dup_rows_remote=18396 dedup_ratio=0.320099 router_tokens_per_expert=[530, 503, 498, 534, 532, 523, 518, 509, 544, 505, 499, 509, 517, 516, 498, 473, 536, 486, 540, 493, 482, 501, 512, 502, 517, 509, 517, 552, 520, 480, 509, 505, 524, 507, 495, 529, 486, 512, 466, 495, 555, 500, 541, 508, 490, 529, 503, 501, 511, 502, 477, 493, 520, 533, 513, 514, 491, 495, 543, 534, 543, 549, 518, 522] payload_rows_per_dst=[5526, 5540, 5614, 5606, 5534, 5584, 5484, 5670] dup_rows_per_dst=[2768, 2582, 2490, 2612, 2494, 2670, 2642, 2720] combine_rows_per_src=[5526, 5540, 5614, 5606, 5534, 5584, 5484, 5670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=38982 dup_rows=20936 dup_rows_remote=18474 dedup_ratio=0.319458 router_tokens_per_expert=[528, 502, 507, 529, 479, 507, 478, 484, 518, 499, 544, 492, 509, 511, 547, 472, 532, 543, 476, 498, 472, 505, 518, 496, 513, 521, 535, 506, 515, 466, 493, 521, 526, 539, 519, 519, 495, 507, 497, 532, 528, 534, 528, 511, 488, 527, 498, 535, 526, 471, 530, 522, 514, 514, 508, 547, 500, 511, 502, 497, 515, 528, 536, 548] payload_rows_per_dst=[5512, 5536, 5618, 5496, 5664, 5638, 5596, 5540] dup_rows_per_dst=[2516, 2648, 2462, 2644, 2604, 2660, 2668, 2734] combine_rows_per_src=[5512, 5536, 5618, 5496, 5664, 5638, 5596, 5540] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=38982 dup_rows=20936 dup_rows_remote=18474 dedup_ratio=0.319458 router_tokens_per_expert=[528, 502, 507, 529, 479, 507, 478, 484, 518, 499, 544, 492, 509, 511, 547, 472, 532, 543, 476, 498, 472, 505, 518, 496, 513, 521, 535, 506, 515, 466, 493, 521, 526, 539, 519, 519, 495, 507, 497, 532, 528, 534, 528, 511, 488, 527, 498, 535, 526, 471, 530, 522, 514, 514, 508, 547, 500, 511, 502, 497, 515, 528, 536, 548] payload_rows_per_dst=[5512, 5536, 5618, 5496, 5664, 5638, 5596, 5540] dup_rows_per_dst=[2516, 2648, 2462, 2644, 2604, 2660, 2668, 2734] combine_rows_per_src=[5512, 5536, 5618, 5496, 5664, 5638, 5596, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=228928 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279580224 ingress_bytes=389451776 link_bytes=389451776 token_bytes=558931520 weight_movement_bytes=110100480 total_nvlink_bytes=669032000 weight_to_token_ratio=0.196984 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773206923e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44280 payload_rows_remote=38628 dup_rows=21256 dup_rows_remote=18456 dedup_ratio=0.324341 router_tokens_per_expert=[511, 500, 533, 535, 517, 530, 485, 521, 489, 504, 488, 534, 514, 482, 468, 512, 511, 517, 531, 523, 521, 541, 527, 507, 461, 505, 516, 547, 555, 508, 537, 499, 529, 532, 543, 546, 539, 508, 517, 512, 485, 511, 507, 472, 559, 459, 523, 483, 522, 485, 523, 521, 503, 495, 508, 509, 537, 474, 496, 492, 517, 480, 522, 530] payload_rows_per_dst=[5588, 5428, 5562, 5560, 5652, 5506, 5524, 5460] dup_rows_per_dst=[2676, 2554, 2794, 2696, 2800, 2492, 2608, 2636] combine_rows_per_src=[5588, 5428, 5562, 5560, 5652, 5506, 5524, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38620 dup_rows=21126 dup_rows_remote=18230 dedup_ratio=0.322357 router_tokens_per_expert=[484, 494, 499, 542, 508, 495, 528, 527, 511, 485, 515, 515, 521, 495, 535, 484, 526, 545, 543, 522, 506, 517, 533, 501, 518, 518, 516, 501, 514, 535, 532, 511, 505, 512, 519, 515, 512, 520, 527, 522, 517, 518, 517, 534, 483, 525, 459, 490, 523, 515, 490, 503, 536, 501, 509, 484, 503, 519, 509, 437, 487, 529, 550, 522] payload_rows_per_dst=[5790, 5510, 5514, 5528, 5578, 5532, 5500, 5458] dup_rows_per_dst=[2896, 2768, 2542, 2576, 2696, 2554, 2622, 2472] combine_rows_per_src=[5790, 5510, 5514, 5528, 5578, 5532, 5500, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277215232 dispatch_meta_bytes=228968 combine_payload_bytes=277215232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277444200 ingress_bytes=387315712 link_bytes=387315712 token_bytes=554659432 weight_movement_bytes=110100480 total_nvlink_bytes=664759912 weight_to_token_ratio=0.198501 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.736057257e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278319104 dispatch_meta_bytes=228888 combine_payload_bytes=278319104 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278547992 ingress_bytes=718721024 link_bytes=718721024 token_bytes=556867096 weight_movement_bytes=440401920 total_nvlink_bytes=997269016 weight_to_token_ratio=0.790856 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249974070e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277688320 dispatch_meta_bytes=228648 combine_payload_bytes=277688320 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=277916968 ingress_bytes=607989760 link_bytes=607989760 token_bytes=555605288 weight_movement_bytes=330301440 total_nvlink_bytes=885906728 weight_to_token_ratio=0.594489 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057394190e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276828160 dispatch_meta_bytes=227400 combine_payload_bytes=276828160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277055560 ingress_bytes=386928640 link_bytes=386928640 token_bytes=553883720 weight_movement_bytes=110100480 total_nvlink_bytes=663984200 weight_to_token_ratio=0.198779 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.729325438e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229824 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279652800 ingress_bytes=499623936 link_bytes=499623936 token_bytes=559075776 weight_movement_bytes=220200960 total_nvlink_bytes=779276736 weight_to_token_ratio=0.393866 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689281987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38896 dup_rows=20988 dup_rows_remote=18256 dedup_ratio=0.320251 router_tokens_per_expert=[478, 450, 530, 489, 516, 499, 495, 529, 532, 505, 545, 505, 510, 519, 513, 465, 485, 539, 519, 528, 505, 506, 465, 479, 510, 535, 514, 556, 537, 487, 476, 515, 549, 514, 496, 519, 501, 509, 566, 538, 536, 508, 490, 498, 505, 529, 534, 517, 524, 484, 551, 511, 492, 530, 539, 510, 530, 504, 514, 499, 498, 503, 540, 494] payload_rows_per_dst=[5470, 5604, 5530, 5504, 5652, 5634, 5642, 5512] dup_rows_per_dst=[2502, 2584, 2522, 2756, 2732, 2600, 2640, 2652] combine_rows_per_src=[5470, 5604, 5530, 5504, 5652, 5634, 5642, 5512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38768 dup_rows=21168 dup_rows_remote=18494 dedup_ratio=0.322998 router_tokens_per_expert=[495, 525, 494, 489, 509, 521, 513, 537, 482, 537, 527, 530, 502, 535, 493, 531, 484, 487, 527, 532, 522, 518, 529, 519, 493, 517, 502, 498, 529, 487, 522, 538, 524, 511, 508, 507, 508, 500, 497, 476, 541, 549, 512, 501, 492, 537, 500, 527, 449, 543, 513, 514, 468, 524, 494, 524, 516, 505, 501, 491, 553, 532, 496, 531] payload_rows_per_dst=[5530, 5600, 5528, 5568, 5522, 5602, 5440, 5578] dup_rows_per_dst=[2636, 2674, 2708, 2604, 2540, 2716, 2618, 2672] combine_rows_per_src=[5530, 5600, 5528, 5568, 5522, 5602, 5440, 5578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44690 payload_rows_remote=39084 dup_rows=20846 dup_rows_remote=18146 dedup_ratio=0.318085 router_tokens_per_expert=[508, 551, 533, 500, 519, 500, 526, 508, 532, 500, 487, 547, 521, 500, 517, 525, 541, 492, 497, 545, 540, 490, 485, 494, 503, 527, 468, 498, 498, 486, 488, 528, 497, 522, 513, 536, 530, 503, 510, 541, 500, 513, 540, 529, 529, 498, 502, 542, 530, 505, 513, 497, 503, 466, 523, 528, 477, 488, 481, 543, 511, 506, 515, 523] payload_rows_per_dst=[5684, 5574, 5526, 5488, 5628, 5606, 5580, 5604] dup_rows_per_dst=[2606, 2684, 2642, 2504, 2676, 2700, 2550, 2484] combine_rows_per_src=[5684, 5574, 5526, 5488, 5628, 5606, 5580, 5604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229656 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279910680 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591704 weight_movement_bytes=110100480 total_nvlink_bytes=669692184 weight_to_token_ratio=0.196751 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276885504 dispatch_meta_bytes=228336 combine_payload_bytes=276885504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277113840 ingress_bytes=386985984 link_bytes=386985984 token_bytes=553999344 weight_movement_bytes=110100480 total_nvlink_bytes=664099824 weight_to_token_ratio=0.198738 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.730322744e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276828160 dispatch_meta_bytes=227400 combine_payload_bytes=276828160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277055560 ingress_bytes=386928640 link_bytes=386928640 token_bytes=553883720 weight_movement_bytes=110100480 total_nvlink_bytes=663984200 weight_to_token_ratio=0.198779 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.729325438e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44434 payload_rows_remote=38956 dup_rows=21102 dup_rows_remote=18404 dedup_ratio=0.321991 router_tokens_per_expert=[518, 490, 517, 479, 533, 513, 493, 541, 539, 507, 497, 502, 501, 524, 543, 483, 535, 490, 498, 484, 476, 504, 573, 531, 521, 526, 517, 508, 511, 550, 493, 558, 511, 501, 484, 533, 524, 497, 514, 544, 503, 486, 482, 461, 519, 522, 491, 482, 492, 504, 497, 501, 534, 491, 555, 514, 510, 562, 509, 527, 524, 529, 491, 519] payload_rows_per_dst=[5562, 5504, 5558, 5634, 5646, 5442, 5478, 5610] dup_rows_per_dst=[2606, 2688, 2624, 2734, 2570, 2450, 2698, 2732] combine_rows_per_src=[5562, 5504, 5558, 5634, 5646, 5442, 5478, 5610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277889024 dispatch_meta_bytes=229048 combine_payload_bytes=277889024 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278118072 ingress_bytes=608190464 link_bytes=608190464 token_bytes=556007096 weight_movement_bytes=330301440 total_nvlink_bytes=886308536 weight_to_token_ratio=0.594060 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057743247e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=228608 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279035136 ingress_bytes=499007488 link_bytes=499007488 token_bytes=557841664 weight_movement_bytes=220200960 total_nvlink_bytes=778042624 weight_to_token_ratio=0.394737 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678560943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38776 dup_rows=21170 dup_rows_remote=18504 dedup_ratio=0.323029 router_tokens_per_expert=[540, 506, 516, 542, 522, 526, 519, 515, 510, 497, 519, 493, 453, 535, 526, 509, 501, 450, 533, 514, 499, 502, 516, 502, 518, 522, 512, 486, 537, 529, 489, 535, 515, 509, 517, 501, 482, 533, 566, 480, 514, 514, 540, 477, 501, 516, 484, 543, 513, 493, 494, 526, 514, 523, 542, 475, 524, 497, 513, 482, 560, 531, 507, 509] payload_rows_per_dst=[5586, 5602, 5526, 5590, 5516, 5482, 5520, 5544] dup_rows_per_dst=[2786, 2482, 2508, 2666, 2690, 2696, 2640, 2702] combine_rows_per_src=[5586, 5602, 5526, 5590, 5516, 5482, 5520, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44318 payload_rows_remote=38802 dup_rows=21218 dup_rows_remote=18624 dedup_ratio=0.323761 router_tokens_per_expert=[525, 498, 504, 528, 500, 501, 529, 476, 505, 539, 515, 540, 502, 494, 510, 492, 509, 508, 515, 516, 496, 483, 517, 538, 506, 511, 476, 521, 547, 535, 551, 517, 519, 492, 522, 475, 548, 472, 539, 520, 523, 508, 546, 523, 489, 528, 480, 497, 526, 493, 504, 551, 490, 525, 502, 546, 485, 520, 514, 513, 499, 504, 503, 508] payload_rows_per_dst=[5496, 5438, 5520, 5628, 5532, 5568, 5620, 5516] dup_rows_per_dst=[2588, 2526, 2644, 2878, 2642, 2602, 2744, 2594] combine_rows_per_src=[5496, 5438, 5520, 5628, 5532, 5568, 5620, 5516] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44400 payload_rows_remote=38718 dup_rows=21136 dup_rows_remote=18412 dedup_ratio=0.322510 router_tokens_per_expert=[493, 546, 562, 533, 521, 474, 549, 520, 530, 516, 506, 528, 521, 485, 486, 561, 495, 535, 518, 499, 525, 468, 517, 508, 498, 457, 482, 518, 517, 511, 492, 516, 518, 547, 514, 535, 487, 510, 508, 508, 518, 536, 518, 536, 507, 511, 498, 498, 504, 497, 512, 546, 498, 480, 511, 522, 537, 472, 492, 494, 506, 512, 532, 517] payload_rows_per_dst=[5682, 5558, 5732, 5406, 5444, 5562, 5486, 5530] dup_rows_per_dst=[2724, 2708, 2850, 2576, 2506, 2672, 2506, 2594] combine_rows_per_src=[5682, 5558, 5732, 5406, 5444, 5562, 5486, 5530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280154112 dispatch_meta_bytes=228920 combine_payload_bytes=280154112 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280383032 ingress_bytes=500355072 link_bytes=500355072 token_bytes=560537144 weight_movement_bytes=220200960 total_nvlink_bytes=780738104 weight_to_token_ratio=0.392839 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701997645e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44690 payload_rows_remote=39084 dup_rows=20846 dup_rows_remote=18146 dedup_ratio=0.318085 router_tokens_per_expert=[508, 551, 533, 500, 519, 500, 526, 508, 532, 500, 487, 547, 521, 500, 517, 525, 541, 492, 497, 545, 540, 490, 485, 494, 503, 527, 468, 498, 498, 486, 488, 528, 497, 522, 513, 536, 530, 503, 510, 541, 500, 513, 540, 529, 529, 498, 502, 542, 530, 505, 513, 497, 503, 466, 523, 528, 477, 488, 481, 543, 511, 506, 515, 523] payload_rows_per_dst=[5684, 5574, 5526, 5488, 5628, 5606, 5580, 5604] dup_rows_per_dst=[2606, 2684, 2642, 2504, 2676, 2700, 2550, 2484] combine_rows_per_src=[5684, 5574, 5526, 5488, 5628, 5606, 5580, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44318 payload_rows_remote=38802 dup_rows=21218 dup_rows_remote=18624 dedup_ratio=0.323761 router_tokens_per_expert=[525, 498, 504, 528, 500, 501, 529, 476, 505, 539, 515, 540, 502, 494, 510, 492, 509, 508, 515, 516, 496, 483, 517, 538, 506, 511, 476, 521, 547, 535, 551, 517, 519, 492, 522, 475, 548, 472, 539, 520, 523, 508, 546, 523, 489, 528, 480, 497, 526, 493, 504, 551, 490, 525, 502, 546, 485, 520, 514, 513, 499, 504, 503, 508] payload_rows_per_dst=[5496, 5438, 5520, 5628, 5532, 5568, 5620, 5516] dup_rows_per_dst=[2588, 2526, 2644, 2878, 2642, 2602, 2744, 2594] combine_rows_per_src=[5496, 5438, 5520, 5628, 5532, 5568, 5620, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38776 dup_rows=21170 dup_rows_remote=18504 dedup_ratio=0.323029 router_tokens_per_expert=[540, 506, 516, 542, 522, 526, 519, 515, 510, 497, 519, 493, 453, 535, 526, 509, 501, 450, 533, 514, 499, 502, 516, 502, 518, 522, 512, 486, 537, 529, 489, 535, 515, 509, 517, 501, 482, 533, 566, 480, 514, 514, 540, 477, 501, 516, 484, 543, 513, 493, 494, 526, 514, 523, 542, 475, 524, 497, 513, 482, 560, 531, 507, 509] payload_rows_per_dst=[5586, 5602, 5526, 5590, 5516, 5482, 5520, 5544] dup_rows_per_dst=[2786, 2482, 2508, 2666, 2690, 2696, 2640, 2702] combine_rows_per_src=[5586, 5602, 5526, 5590, 5516, 5482, 5520, 5544] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229440 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279466048 ingress_bytes=499437568 link_bytes=499437568 token_bytes=558702656 weight_movement_bytes=220200960 total_nvlink_bytes=778903616 weight_to_token_ratio=0.394129 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686040741e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=39046 dup_rows=20926 dup_rows_remote=18372 dedup_ratio=0.319305 router_tokens_per_expert=[491, 520, 527, 527, 498, 468, 492, 516, 530, 500, 522, 518, 482, 550, 540, 513, 501, 530, 494, 493, 508, 519, 494, 520, 476, 486, 515, 487, 534, 487, 526, 489, 521, 533, 519, 503, 513, 524, 485, 522, 520, 551, 537, 481, 520, 519, 486, 488, 502, 513, 499, 531, 537, 541, 516, 548, 516, 516, 547, 484, 519, 502, 540, 482] payload_rows_per_dst=[5486, 5600, 5564, 5510, 5596, 5574, 5670, 5610] dup_rows_per_dst=[2592, 2710, 2554, 2490, 2644, 2630, 2704, 2602] combine_rows_per_src=[5486, 5600, 5564, 5510, 5596, 5574, 5670, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44400 payload_rows_remote=38718 dup_rows=21136 dup_rows_remote=18412 dedup_ratio=0.322510 router_tokens_per_expert=[493, 546, 562, 533, 521, 474, 549, 520, 530, 516, 506, 528, 521, 485, 486, 561, 495, 535, 518, 499, 525, 468, 517, 508, 498, 457, 482, 518, 517, 511, 492, 516, 518, 547, 514, 535, 487, 510, 508, 508, 518, 536, 518, 536, 507, 511, 498, 498, 504, 497, 512, 546, 498, 480, 511, 522, 537, 472, 492, 494, 506, 512, 532, 517] payload_rows_per_dst=[5682, 5558, 5732, 5406, 5444, 5562, 5486, 5530] dup_rows_per_dst=[2724, 2708, 2850, 2576, 2506, 2672, 2506, 2594] combine_rows_per_src=[5682, 5558, 5732, 5406, 5444, 5562, 5486, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38768 dup_rows=21168 dup_rows_remote=18494 dedup_ratio=0.322998 router_tokens_per_expert=[495, 525, 494, 489, 509, 521, 513, 537, 482, 537, 527, 530, 502, 535, 493, 531, 484, 487, 527, 532, 522, 518, 529, 519, 493, 517, 502, 498, 529, 487, 522, 538, 524, 511, 508, 507, 508, 500, 497, 476, 541, 549, 512, 501, 492, 537, 500, 527, 449, 543, 513, 514, 468, 524, 494, 524, 516, 505, 501, 491, 553, 532, 496, 531] payload_rows_per_dst=[5530, 5600, 5528, 5568, 5522, 5602, 5440, 5578] dup_rows_per_dst=[2636, 2674, 2708, 2604, 2540, 2716, 2618, 2672] combine_rows_per_src=[5530, 5600, 5528, 5568, 5522, 5602, 5440, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38896 dup_rows=20988 dup_rows_remote=18256 dedup_ratio=0.320251 router_tokens_per_expert=[478, 450, 530, 489, 516, 499, 495, 529, 532, 505, 545, 505, 510, 519, 513, 465, 485, 539, 519, 528, 505, 506, 465, 479, 510, 535, 514, 556, 537, 487, 476, 515, 549, 514, 496, 519, 501, 509, 566, 538, 536, 508, 490, 498, 505, 529, 534, 517, 524, 484, 551, 511, 492, 530, 539, 510, 530, 504, 514, 499, 498, 503, 540, 494] payload_rows_per_dst=[5470, 5604, 5530, 5504, 5652, 5634, 5642, 5512] dup_rows_per_dst=[2502, 2584, 2522, 2756, 2732, 2600, 2640, 2652] combine_rows_per_src=[5470, 5604, 5530, 5504, 5652, 5634, 5642, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280154112 dispatch_meta_bytes=228920 combine_payload_bytes=280154112 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280383032 ingress_bytes=500355072 link_bytes=500355072 token_bytes=560537144 weight_movement_bytes=220200960 total_nvlink_bytes=780738104 weight_to_token_ratio=0.392839 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701997645e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=229120 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278175488 ingress_bytes=388046848 link_bytes=388046848 token_bytes=556121856 weight_movement_bytes=110100480 total_nvlink_bytes=666222336 weight_to_token_ratio=0.197979 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748772914e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278132736 dispatch_meta_bytes=229704 combine_payload_bytes=278132736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278362440 ingress_bytes=388233216 link_bytes=388233216 token_bytes=556495176 weight_movement_bytes=110100480 total_nvlink_bytes=666595656 weight_to_token_ratio=0.197846 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752014160e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277530624 dispatch_meta_bytes=228520 combine_payload_bytes=277530624 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277759144 ingress_bytes=497731584 link_bytes=497731584 token_bytes=555289768 weight_movement_bytes=220200960 total_nvlink_bytes=775490728 weight_to_token_ratio=0.396551 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.656370874e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38826 dup_rows=21160 dup_rows_remote=18542 dedup_ratio=0.322876 router_tokens_per_expert=[510, 511, 515, 536, 515, 502, 534, 515, 523, 477, 535, 523, 487, 510, 513, 516, 519, 537, 525, 491, 514, 482, 555, 553, 502, 522, 531, 505, 515, 511, 499, 516, 529, 492, 449, 508, 462, 527, 526, 509, 527, 459, 481, 525, 511, 523, 535, 516, 499, 528, 493, 504, 522, 509, 553, 528, 533, 512, 511, 489, 533, 487, 507, 482] payload_rows_per_dst=[5540, 5550, 5640, 5552, 5490, 5556, 5546, 5502] dup_rows_per_dst=[2736, 2618, 2712, 2650, 2514, 2598, 2726, 2606] combine_rows_per_src=[5540, 5550, 5640, 5552, 5490, 5556, 5546, 5502] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44324 payload_rows_remote=38776 dup_rows=21212 dup_rows_remote=18464 dedup_ratio=0.323669 router_tokens_per_expert=[498, 482, 522, 516, 510, 541, 500, 496, 540, 528, 501, 505, 509, 528, 535, 494, 502, 521, 538, 480, 503, 543, 518, 537, 470, 523, 524, 508, 516, 498, 488, 522, 496, 493, 471, 558, 550, 524, 545, 511, 485, 512, 511, 513, 508, 486, 488, 493, 482, 508, 524, 477, 527, 536, 513, 503, 456, 548, 523, 513, 540, 515, 534, 529] payload_rows_per_dst=[5536, 5604, 5612, 5486, 5548, 5404, 5544, 5590] dup_rows_per_dst=[2594, 2676, 2672, 2612, 2748, 2588, 2596, 2726] combine_rows_per_src=[5536, 5604, 5612, 5486, 5548, 5404, 5544, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=229120 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278175488 ingress_bytes=388046848 link_bytes=388046848 token_bytes=556121856 weight_movement_bytes=110100480 total_nvlink_bytes=666222336 weight_to_token_ratio=0.197979 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748772914e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=229672 combine_payload_bytes=279881728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280111400 ingress_bytes=389982208 link_bytes=389982208 token_bytes=559993128 weight_movement_bytes=110100480 total_nvlink_bytes=670093608 weight_to_token_ratio=0.196610 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782432008e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44704 payload_rows_remote=39104 dup_rows=20832 dup_rows_remote=18110 dedup_ratio=0.317871 router_tokens_per_expert=[502, 536, 535, 487, 492, 522, 497, 510, 508, 494, 546, 535, 546, 505, 496, 491, 533, 492, 510, 469, 505, 506, 496, 495, 540, 488, 528, 513, 500, 491, 512, 555, 505, 483, 496, 522, 517, 542, 523, 523, 558, 517, 529, 511, 523, 505, 517, 501, 541, 480, 491, 531, 509, 506, 501, 496, 532, 482, 525, 520, 514, 513, 494, 526] payload_rows_per_dst=[5592, 5664, 5484, 5692, 5570, 5600, 5514, 5588] dup_rows_per_dst=[2570, 2578, 2528, 2562, 2652, 2722, 2596, 2624] combine_rows_per_src=[5592, 5664, 5484, 5692, 5570, 5600, 5514, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277530624 dispatch_meta_bytes=228520 combine_payload_bytes=277530624 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277759144 ingress_bytes=497731584 link_bytes=497731584 token_bytes=555289768 weight_movement_bytes=220200960 total_nvlink_bytes=775490728 weight_to_token_ratio=0.396551 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.656370874e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277889024 dispatch_meta_bytes=229048 combine_payload_bytes=277889024 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278118072 ingress_bytes=608190464 link_bytes=608190464 token_bytes=556007096 weight_movement_bytes=330301440 total_nvlink_bytes=886308536 weight_to_token_ratio=0.594060 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057743247e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=228608 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279035136 ingress_bytes=499007488 link_bytes=499007488 token_bytes=557841664 weight_movement_bytes=220200960 total_nvlink_bytes=778042624 weight_to_token_ratio=0.394737 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678560943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44288 payload_rows_remote=38844 dup_rows=21248 dup_rows_remote=18634 dedup_ratio=0.324219 router_tokens_per_expert=[520, 526, 506, 523, 503, 531, 521, 496, 477, 485, 561, 499, 526, 498, 558, 520, 545, 490, 512, 528, 496, 508, 535, 503, 481, 527, 524, 506, 504, 510, 509, 483, 539, 478, 511, 500, 481, 541, 507, 531, 525, 523, 510, 514, 551, 520, 481, 487, 499, 503, 518, 489, 490, 479, 547, 504, 499, 557, 470, 530, 498, 517, 517, 541] payload_rows_per_dst=[5560, 5584, 5582, 5510, 5488, 5536, 5444, 5584] dup_rows_per_dst=[2692, 2664, 2652, 2578, 2688, 2686, 2614, 2674] combine_rows_per_src=[5560, 5584, 5582, 5510, 5488, 5536, 5444, 5584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229472 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278534240 ingress_bytes=498505728 link_bytes=498505728 token_bytes=556839008 weight_movement_bytes=220200960 total_nvlink_bytes=777039968 weight_to_token_ratio=0.395448 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669834511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228960 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278175328 ingress_bytes=388046848 link_bytes=388046848 token_bytes=556121696 weight_movement_bytes=110100480 total_nvlink_bytes=666222176 weight_to_token_ratio=0.197979 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748772914e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=39030 dup_rows=21036 dup_rows_remote=18490 dedup_ratio=0.320984 router_tokens_per_expert=[508, 504, 547, 521, 526, 508, 535, 540, 487, 524, 538, 532, 493, 525, 579, 555, 518, 449, 498, 540, 540, 499, 523, 540, 502, 474, 498, 478, 545, 518, 489, 504, 537, 505, 485, 514, 486, 494, 536, 507, 520, 506, 503, 523, 517, 546, 483, 510, 470, 507, 542, 515, 499, 514, 530, 500, 499, 475, 512, 483, 504, 514, 502, 493] payload_rows_per_dst=[5674, 5664, 5622, 5470, 5524, 5556, 5540, 5450] dup_rows_per_dst=[2704, 2802, 2592, 2546, 2604, 2660, 2614, 2514] combine_rows_per_src=[5674, 5664, 5622, 5470, 5524, 5556, 5540, 5450] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44704 payload_rows_remote=39104 dup_rows=20832 dup_rows_remote=18110 dedup_ratio=0.317871 router_tokens_per_expert=[502, 536, 535, 487, 492, 522, 497, 510, 508, 494, 546, 535, 546, 505, 496, 491, 533, 492, 510, 469, 505, 506, 496, 495, 540, 488, 528, 513, 500, 491, 512, 555, 505, 483, 496, 522, 517, 542, 523, 523, 558, 517, 529, 511, 523, 505, 517, 501, 541, 480, 491, 531, 509, 506, 501, 496, 532, 482, 525, 520, 514, 513, 494, 526] payload_rows_per_dst=[5592, 5664, 5484, 5692, 5570, 5600, 5514, 5588] dup_rows_per_dst=[2570, 2578, 2528, 2562, 2652, 2722, 2596, 2624] combine_rows_per_src=[5592, 5664, 5484, 5692, 5570, 5600, 5514, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=229912 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278663704 ingress_bytes=388534272 link_bytes=388534272 token_bytes=557097496 weight_movement_bytes=110100480 total_nvlink_bytes=667197976 weight_to_token_ratio=0.197632 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757250019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44358 payload_rows_remote=38766 dup_rows=21178 dup_rows_remote=18386 dedup_ratio=0.323151 router_tokens_per_expert=[490, 532, 531, 504, 552, 498, 514, 523, 529, 523, 507, 522, 518, 522, 538, 486, 524, 547, 521, 484, 517, 516, 537, 537, 524, 480, 514, 483, 509, 493, 473, 490, 528, 505, 471, 514, 463, 525, 490, 452, 503, 563, 545, 483, 496, 508, 513, 495, 499, 521, 522, 528, 507, 495, 489, 523, 509, 529, 488, 568, 535, 509, 528, 526] payload_rows_per_dst=[5400, 5292, 5522, 5720, 5578, 5590, 5664, 5592] dup_rows_per_dst=[2388, 2368, 2576, 2842, 2818, 2622, 2772, 2792] combine_rows_per_src=[5400, 5292, 5522, 5720, 5578, 5590, 5664, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=39030 dup_rows=21036 dup_rows_remote=18490 dedup_ratio=0.320984 router_tokens_per_expert=[508, 504, 547, 521, 526, 508, 535, 540, 487, 524, 538, 532, 493, 525, 579, 555, 518, 449, 498, 540, 540, 499, 523, 540, 502, 474, 498, 478, 545, 518, 489, 504, 537, 505, 485, 514, 486, 494, 536, 507, 520, 506, 503, 523, 517, 546, 483, 510, 470, 507, 542, 515, 499, 514, 530, 500, 499, 475, 512, 483, 504, 514, 502, 493] payload_rows_per_dst=[5674, 5664, 5622, 5470, 5524, 5556, 5540, 5450] dup_rows_per_dst=[2704, 2802, 2592, 2546, 2604, 2660, 2614, 2514] combine_rows_per_src=[5674, 5664, 5622, 5470, 5524, 5556, 5540, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38780 dup_rows=21146 dup_rows_remote=18404 dedup_ratio=0.322662 router_tokens_per_expert=[530, 521, 494, 513, 483, 541, 523, 515, 535, 533, 487, 496, 497, 513, 507, 512, 547, 504, 512, 524, 553, 489, 566, 481, 520, 467, 539, 504, 516, 498, 528, 511, 488, 548, 561, 503, 487, 484, 513, 491, 519, 521, 511, 520, 506, 493, 477, 515, 500, 522, 514, 513, 550, 503, 528, 485, 483, 520, 520, 486, 523, 540, 475, 510] payload_rows_per_dst=[5624, 5478, 5610, 5588, 5508, 5506, 5518, 5558] dup_rows_per_dst=[2616, 2682, 2742, 2578, 2642, 2618, 2712, 2556] combine_rows_per_src=[5624, 5478, 5610, 5588, 5508, 5506, 5518, 5558] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44240 payload_rows_remote=38662 dup_rows=21296 dup_rows_remote=18564 dedup_ratio=0.324951 router_tokens_per_expert=[512, 535, 505, 483, 536, 504, 498, 535, 528, 514, 487, 528, 511, 502, 520, 501, 534, 500, 496, 532, 511, 519, 488, 479, 493, 485, 529, 485, 541, 530, 499, 513, 520, 479, 541, 507, 546, 495, 499, 495, 491, 543, 533, 545, 507, 506, 525, 522, 537, 510, 457, 534, 524, 507, 487, 512, 533, 528, 516, 479, 515, 528, 520, 494] payload_rows_per_dst=[5578, 5534, 5472, 5470, 5574, 5576, 5532, 5504] dup_rows_per_dst=[2732, 2778, 2518, 2680, 2590, 2718, 2604, 2676] combine_rows_per_src=[5578, 5534, 5472, 5470, 5574, 5576, 5532, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38826 dup_rows=21160 dup_rows_remote=18542 dedup_ratio=0.322876 router_tokens_per_expert=[510, 511, 515, 536, 515, 502, 534, 515, 523, 477, 535, 523, 487, 510, 513, 516, 519, 537, 525, 491, 514, 482, 555, 553, 502, 522, 531, 505, 515, 511, 499, 516, 529, 492, 449, 508, 462, 527, 526, 509, 527, 459, 481, 525, 511, 523, 535, 516, 499, 528, 493, 504, 522, 509, 553, 528, 533, 512, 511, 489, 533, 487, 507, 482] payload_rows_per_dst=[5540, 5550, 5640, 5552, 5490, 5556, 5546, 5502] dup_rows_per_dst=[2736, 2618, 2712, 2650, 2514, 2598, 2726, 2606] combine_rows_per_src=[5540, 5550, 5640, 5552, 5490, 5556, 5546, 5502] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280297472 dispatch_meta_bytes=228856 combine_payload_bytes=280297472 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280526328 ingress_bytes=500498432 link_bytes=500498432 token_bytes=560823800 weight_movement_bytes=220200960 total_nvlink_bytes=781024760 weight_to_token_ratio=0.392638 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.704490911e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=229912 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278663704 ingress_bytes=388534272 link_bytes=388534272 token_bytes=557097496 weight_movement_bytes=110100480 total_nvlink_bytes=667197976 weight_to_token_ratio=0.197632 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757250019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44324 payload_rows_remote=38776 dup_rows=21212 dup_rows_remote=18464 dedup_ratio=0.323669 router_tokens_per_expert=[498, 482, 522, 516, 510, 541, 500, 496, 540, 528, 501, 505, 509, 528, 535, 494, 502, 521, 538, 480, 503, 543, 518, 537, 470, 523, 524, 508, 516, 498, 488, 522, 496, 493, 471, 558, 550, 524, 545, 511, 485, 512, 511, 513, 508, 486, 488, 493, 482, 508, 524, 477, 527, 536, 513, 503, 456, 548, 523, 513, 540, 515, 534, 529] payload_rows_per_dst=[5536, 5604, 5612, 5486, 5548, 5404, 5544, 5590] dup_rows_per_dst=[2594, 2676, 2672, 2612, 2748, 2588, 2596, 2726] combine_rows_per_src=[5536, 5604, 5612, 5486, 5548, 5404, 5544, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38780 dup_rows=21146 dup_rows_remote=18404 dedup_ratio=0.322662 router_tokens_per_expert=[530, 521, 494, 513, 483, 541, 523, 515, 535, 533, 487, 496, 497, 513, 507, 512, 547, 504, 512, 524, 553, 489, 566, 481, 520, 467, 539, 504, 516, 498, 528, 511, 488, 548, 561, 503, 487, 484, 513, 491, 519, 521, 511, 520, 506, 493, 477, 515, 500, 522, 514, 513, 550, 503, 528, 485, 483, 520, 520, 486, 523, 540, 475, 510] payload_rows_per_dst=[5624, 5478, 5610, 5588, 5508, 5506, 5518, 5558] dup_rows_per_dst=[2616, 2682, 2742, 2578, 2642, 2618, 2712, 2556] combine_rows_per_src=[5624, 5478, 5610, 5588, 5508, 5506, 5518, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=230080 combine_payload_bytes=279767040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279997120 ingress_bytes=389867520 link_bytes=389867520 token_bytes=559764160 weight_movement_bytes=110100480 total_nvlink_bytes=669864640 weight_to_token_ratio=0.196691 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780437395e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277874688 dispatch_meta_bytes=228608 combine_payload_bytes=277874688 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278103296 ingress_bytes=498075648 link_bytes=498075648 token_bytes=555977984 weight_movement_bytes=220200960 total_nvlink_bytes=776178944 weight_to_token_ratio=0.396061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662354712e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277129216 dispatch_meta_bytes=228904 combine_payload_bytes=277129216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277358120 ingress_bytes=497330176 link_bytes=497330176 token_bytes=554487336 weight_movement_bytes=220200960 total_nvlink_bytes=774688296 weight_to_token_ratio=0.397125 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.649389728e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44360 payload_rows_remote=38780 dup_rows=21176 dup_rows_remote=18414 dedup_ratio=0.323120 router_tokens_per_expert=[455, 535, 490, 541, 515, 525, 516, 503, 516, 510, 515, 525, 548, 513, 516, 528, 537, 542, 490, 490, 514, 511, 519, 452, 543, 504, 518, 505, 543, 537, 517, 523, 517, 498, 509, 489, 476, 504, 534, 471, 534, 506, 517, 501, 528, 535, 505, 543, 531, 530, 500, 511, 506, 477, 539, 503, 526, 468, 508, 491, 492, 480, 509, 534] payload_rows_per_dst=[5536, 5580, 5560, 5700, 5454, 5558, 5508, 5464] dup_rows_per_dst=[2624, 2762, 2550, 2680, 2542, 2780, 2686, 2552] combine_rows_per_src=[5536, 5580, 5560, 5700, 5454, 5558, 5508, 5464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38850 dup_rows=21092 dup_rows_remote=18494 dedup_ratio=0.321838 router_tokens_per_expert=[505, 505, 505, 504, 558, 542, 509, 562, 513, 500, 491, 486, 492, 511, 492, 481, 477, 531, 548, 515, 506, 487, 487, 512, 517, 538, 522, 504, 484, 520, 483, 524, 511, 528, 525, 505, 533, 517, 516, 461, 493, 523, 499, 529, 537, 528, 541, 472, 497, 498, 503, 468, 506, 498, 527, 529, 552, 555, 472, 545, 538, 516, 500, 535] payload_rows_per_dst=[5616, 5448, 5554, 5478, 5594, 5616, 5446, 5692] dup_rows_per_dst=[2764, 2484, 2572, 2706, 2598, 2628, 2606, 2734] combine_rows_per_src=[5616, 5448, 5554, 5478, 5594, 5616, 5446, 5692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=230080 combine_payload_bytes=279767040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279997120 ingress_bytes=389867520 link_bytes=389867520 token_bytes=559764160 weight_movement_bytes=110100480 total_nvlink_bytes=669864640 weight_to_token_ratio=0.196691 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780437395e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=228736 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278203776 ingress_bytes=608276480 link_bytes=608276480 token_bytes=556178816 weight_movement_bytes=330301440 total_nvlink_bytes=886480256 weight_to_token_ratio=0.593876 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057892843e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38992 dup_rows=20992 dup_rows_remote=18316 dedup_ratio=0.320312 router_tokens_per_expert=[517, 519, 519, 556, 471, 495, 543, 527, 486, 506, 550, 487, 484, 514, 475, 522, 517, 490, 557, 497, 486, 503, 504, 466, 529, 505, 492, 502, 526, 500, 513, 496, 514, 534, 544, 517, 543, 531, 505, 523, 543, 486, 495, 558, 527, 502, 493, 510, 529, 531, 499, 487, 497, 537, 544, 499, 514, 517, 511, 537, 491, 492, 507, 497] payload_rows_per_dst=[5684, 5508, 5532, 5538, 5594, 5552, 5606, 5530] dup_rows_per_dst=[2610, 2540, 2508, 2588, 2828, 2676, 2640, 2602] combine_rows_per_src=[5684, 5508, 5532, 5538, 5594, 5552, 5606, 5530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=228776 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278203816 ingress_bytes=388075520 link_bytes=388075520 token_bytes=556178856 weight_movement_bytes=110100480 total_nvlink_bytes=666279336 weight_to_token_ratio=0.197959 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749271567e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=229376 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278706176 ingress_bytes=388577280 link_bytes=388577280 token_bytes=557182976 weight_movement_bytes=110100480 total_nvlink_bytes=667283456 weight_to_token_ratio=0.197602 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757997999e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277129216 dispatch_meta_bytes=228904 combine_payload_bytes=277129216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277358120 ingress_bytes=497330176 link_bytes=497330176 token_bytes=554487336 weight_movement_bytes=220200960 total_nvlink_bytes=774688296 weight_to_token_ratio=0.397125 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.649389728e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229472 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278534240 ingress_bytes=498505728 link_bytes=498505728 token_bytes=556839008 weight_movement_bytes=220200960 total_nvlink_bytes=777039968 weight_to_token_ratio=0.395448 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669834511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44606 payload_rows_remote=39116 dup_rows=20930 dup_rows_remote=18320 dedup_ratio=0.319366 router_tokens_per_expert=[526, 521, 481, 510, 508, 475, 520, 552, 509, 491, 517, 521, 544, 496, 563, 497, 506, 507, 497, 508, 486, 520, 511, 553, 504, 506, 475, 510, 528, 504, 520, 506, 500, 513, 501, 514, 508, 522, 455, 507, 518, 535, 529, 532, 517, 540, 556, 517, 480, 518, 540, 502, 484, 497, 488, 541, 529, 488, 506, 524, 510, 490, 515, 520] payload_rows_per_dst=[5536, 5612, 5572, 5584, 5516, 5710, 5490, 5586] dup_rows_per_dst=[2650, 2664, 2604, 2522, 2524, 2778, 2610, 2578] combine_rows_per_src=[5536, 5612, 5572, 5584, 5516, 5710, 5490, 5586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228960 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278175328 ingress_bytes=388046848 link_bytes=388046848 token_bytes=556121696 weight_movement_bytes=110100480 total_nvlink_bytes=666222176 weight_to_token_ratio=0.197979 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748772914e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38992 dup_rows=20992 dup_rows_remote=18316 dedup_ratio=0.320312 router_tokens_per_expert=[517, 519, 519, 556, 471, 495, 543, 527, 486, 506, 550, 487, 484, 514, 475, 522, 517, 490, 557, 497, 486, 503, 504, 466, 529, 505, 492, 502, 526, 500, 513, 496, 514, 534, 544, 517, 543, 531, 505, 523, 543, 486, 495, 558, 527, 502, 493, 510, 529, 531, 499, 487, 497, 537, 544, 499, 514, 517, 511, 537, 491, 492, 507, 497] payload_rows_per_dst=[5684, 5508, 5532, 5538, 5594, 5552, 5606, 5530] dup_rows_per_dst=[2610, 2540, 2508, 2588, 2828, 2676, 2640, 2602] combine_rows_per_src=[5684, 5508, 5532, 5538, 5594, 5552, 5606, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280383488 dispatch_meta_bytes=229744 combine_payload_bytes=280383488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280613232 ingress_bytes=390483968 link_bytes=390483968 token_bytes=560996720 weight_movement_bytes=110100480 total_nvlink_bytes=671097200 weight_to_token_ratio=0.196259 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.791158439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44720 payload_rows_remote=39142 dup_rows=20816 dup_rows_remote=18266 dedup_ratio=0.317627 router_tokens_per_expert=[511, 552, 532, 489, 514, 521, 488, 513, 506, 525, 542, 494, 489, 514, 514, 511, 547, 504, 524, 542, 544, 499, 484, 516, 516, 483, 486, 529, 511, 499, 521, 476, 522, 496, 526, 515, 500, 517, 531, 537, 498, 504, 522, 517, 507, 512, 511, 467, 537, 494, 512, 523, 502, 519, 582, 519, 527, 490, 498, 468, 513, 502, 511, 493] payload_rows_per_dst=[5438, 5588, 5688, 5594, 5602, 5518, 5714, 5578] dup_rows_per_dst=[2430, 2602, 2768, 2652, 2686, 2436, 2692, 2550] combine_rows_per_src=[5438, 5588, 5688, 5594, 5602, 5518, 5714, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38896 dup_rows=21086 dup_rows_remote=18468 dedup_ratio=0.321747 router_tokens_per_expert=[515, 486, 525, 565, 511, 504, 505, 553, 543, 538, 512, 513, 510, 501, 489, 523, 548, 507, 486, 514, 516, 477, 517, 530, 525, 528, 509, 539, 512, 472, 500, 501, 529, 481, 540, 511, 511, 542, 503, 519, 483, 492, 521, 508, 513, 476, 495, 555, 498, 484, 518, 518, 480, 487, 516, 516, 540, 515, 502, 478, 546, 520, 527, 470] payload_rows_per_dst=[5636, 5518, 5576, 5554, 5590, 5512, 5480, 5584] dup_rows_per_dst=[2692, 2740, 2614, 2618, 2682, 2574, 2554, 2612] combine_rows_per_src=[5636, 5518, 5576, 5554, 5590, 5512, 5480, 5584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44632 payload_rows_remote=38996 dup_rows=20904 dup_rows_remote=18282 dedup_ratio=0.318970 router_tokens_per_expert=[545, 531, 501, 503, 538, 502, 505, 504, 501, 484, 556, 505, 471, 538, 533, 485, 535, 517, 509, 482, 511, 496, 513, 506, 490, 507, 516, 534, 520, 524, 501, 528, 555, 520, 503, 492, 517, 550, 486, 503, 460, 517, 529, 541, 525, 511, 468, 538, 508, 491, 487, 517, 503, 561, 504, 510, 517, 499, 504, 501, 517, 491, 542, 510] payload_rows_per_dst=[5636, 5628, 5614, 5554, 5560, 5506, 5592, 5542] dup_rows_per_dst=[2622, 2660, 2524, 2686, 2714, 2508, 2570, 2620] combine_rows_per_src=[5636, 5628, 5614, 5554, 5560, 5506, 5592, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279494656 dispatch_meta_bytes=229232 combine_payload_bytes=279494656 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279723888 ingress_bytes=719896576 link_bytes=719896576 token_bytes=559218544 weight_movement_bytes=440401920 total_nvlink_bytes=999620464 weight_to_token_ratio=0.787531 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252018548e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280383488 dispatch_meta_bytes=229744 combine_payload_bytes=280383488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280613232 ingress_bytes=390483968 link_bytes=390483968 token_bytes=560996720 weight_movement_bytes=110100480 total_nvlink_bytes=671097200 weight_to_token_ratio=0.196259 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.791158439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=39000 dup_rows=20918 dup_rows_remote=18236 dedup_ratio=0.319183 router_tokens_per_expert=[487, 512, 487, 532, 503, 486, 513, 511, 509, 521, 481, 511, 506, 529, 516, 528, 536, 546, 504, 521, 525, 490, 515, 513, 513, 505, 489, 511, 495, 473, 518, 537, 513, 475, 495, 495, 487, 512, 533, 516, 512, 515, 520, 519, 465, 527, 527, 507, 530, 526, 514, 482, 539, 540, 511, 529, 531, 491, 502, 525, 535, 539, 530, 503] payload_rows_per_dst=[5534, 5562, 5618, 5546, 5506, 5584, 5656, 5612] dup_rows_per_dst=[2528, 2640, 2682, 2536, 2546, 2600, 2686, 2700] combine_rows_per_src=[5534, 5562, 5618, 5546, 5506, 5584, 5656, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279494656 dispatch_meta_bytes=229232 combine_payload_bytes=279494656 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279723888 ingress_bytes=719896576 link_bytes=719896576 token_bytes=559218544 weight_movement_bytes=440401920 total_nvlink_bytes=999620464 weight_to_token_ratio=0.787531 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252018548e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44632 payload_rows_remote=38996 dup_rows=20904 dup_rows_remote=18282 dedup_ratio=0.318970 router_tokens_per_expert=[545, 531, 501, 503, 538, 502, 505, 504, 501, 484, 556, 505, 471, 538, 533, 485, 535, 517, 509, 482, 511, 496, 513, 506, 490, 507, 516, 534, 520, 524, 501, 528, 555, 520, 503, 492, 517, 550, 486, 503, 460, 517, 529, 541, 525, 511, 468, 538, 508, 491, 487, 517, 503, 561, 504, 510, 517, 499, 504, 501, 517, 491, 542, 510] payload_rows_per_dst=[5636, 5628, 5614, 5554, 5560, 5506, 5592, 5542] dup_rows_per_dst=[2622, 2660, 2524, 2686, 2714, 2508, 2570, 2620] combine_rows_per_src=[5636, 5628, 5614, 5554, 5560, 5506, 5592, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44360 payload_rows_remote=38780 dup_rows=21176 dup_rows_remote=18414 dedup_ratio=0.323120 router_tokens_per_expert=[455, 535, 490, 541, 515, 525, 516, 503, 516, 510, 515, 525, 548, 513, 516, 528, 537, 542, 490, 490, 514, 511, 519, 452, 543, 504, 518, 505, 543, 537, 517, 523, 517, 498, 509, 489, 476, 504, 534, 471, 534, 506, 517, 501, 528, 535, 505, 543, 531, 530, 500, 511, 506, 477, 539, 503, 526, 468, 508, 491, 492, 480, 509, 534] payload_rows_per_dst=[5536, 5580, 5560, 5700, 5454, 5558, 5508, 5464] dup_rows_per_dst=[2624, 2762, 2550, 2680, 2542, 2780, 2686, 2552] combine_rows_per_src=[5536, 5580, 5560, 5700, 5454, 5558, 5508, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38850 dup_rows=21092 dup_rows_remote=18494 dedup_ratio=0.321838 router_tokens_per_expert=[505, 505, 505, 504, 558, 542, 509, 562, 513, 500, 491, 486, 492, 511, 492, 481, 477, 531, 548, 515, 506, 487, 487, 512, 517, 538, 522, 504, 484, 520, 483, 524, 511, 528, 525, 505, 533, 517, 516, 461, 493, 523, 499, 529, 537, 528, 541, 472, 497, 498, 503, 468, 506, 498, 527, 529, 552, 555, 472, 545, 538, 516, 500, 535] payload_rows_per_dst=[5616, 5448, 5554, 5478, 5594, 5616, 5446, 5692] dup_rows_per_dst=[2764, 2484, 2572, 2706, 2598, 2628, 2606, 2734] combine_rows_per_src=[5616, 5448, 5554, 5478, 5594, 5616, 5446, 5692] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280569856 dispatch_meta_bytes=229632 combine_payload_bytes=280569856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280799488 ingress_bytes=390670336 link_bytes=390670336 token_bytes=561369344 weight_movement_bytes=110100480 total_nvlink_bytes=671469824 weight_to_token_ratio=0.196128 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794399685e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=229456 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279035984 ingress_bytes=388907008 link_bytes=388907008 token_bytes=557842512 weight_movement_bytes=110100480 total_nvlink_bytes=667942992 weight_to_token_ratio=0.197368 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763732511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=229112 combine_payload_bytes=279523328 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=279752440 ingress_bytes=719925248 link_bytes=719925248 token_bytes=559275768 weight_movement_bytes=440401920 total_nvlink_bytes=999677688 weight_to_token_ratio=0.787450 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252068414e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=39016 dup_rows=21042 dup_rows_remote=18396 dedup_ratio=0.321075 router_tokens_per_expert=[517, 503, 507, 504, 535, 514, 485, 510, 493, 505, 525, 542, 516, 503, 482, 496, 523, 489, 501, 499, 512, 497, 494, 535, 546, 519, 537, 505, 522, 520, 496, 506, 514, 519, 518, 525, 518, 508, 559, 552, 562, 550, 483, 511, 536, 518, 483, 476, 550, 490, 517, 522, 509, 495, 503, 495, 497, 493, 484, 532, 478, 517, 525, 491] payload_rows_per_dst=[5522, 5478, 5492, 5626, 5688, 5608, 5576, 5504] dup_rows_per_dst=[2628, 2646, 2608, 2676, 2738, 2630, 2586, 2530] combine_rows_per_src=[5522, 5478, 5492, 5626, 5688, 5608, 5576, 5504] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44554 payload_rows_remote=39040 dup_rows=20982 dup_rows_remote=18454 dedup_ratio=0.320160 router_tokens_per_expert=[504, 517, 534, 508, 536, 525, 536, 534, 568, 512, 505, 516, 492, 526, 509, 509, 503, 477, 481, 533, 503, 477, 523, 515, 469, 506, 535, 507, 524, 532, 478, 546, 497, 478, 526, 522, 492, 492, 511, 503, 536, 521, 541, 528, 479, 504, 485, 528, 484, 525, 528, 523, 494, 502, 536, 514, 508, 497, 513, 502, 511, 494, 539, 515] payload_rows_per_dst=[5620, 5620, 5482, 5598, 5514, 5582, 5588, 5550] dup_rows_per_dst=[2768, 2654, 2542, 2596, 2528, 2662, 2624, 2608] combine_rows_per_src=[5620, 5620, 5482, 5598, 5514, 5582, 5588, 5550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=228944 combine_payload_bytes=279552000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279780944 ingress_bytes=389652480 link_bytes=389652480 token_bytes=559332944 weight_movement_bytes=110100480 total_nvlink_bytes=669433424 weight_to_token_ratio=0.196842 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776697495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=229648 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279896336 ingress_bytes=499867648 link_bytes=499867648 token_bytes=559563024 weight_movement_bytes=220200960 total_nvlink_bytes=779763984 weight_to_token_ratio=0.393523 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693520540e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38772 dup_rows=21116 dup_rows_remote=18498 dedup_ratio=0.322205 router_tokens_per_expert=[510, 512, 530, 514, 497, 526, 528, 512, 509, 515, 502, 494, 534, 515, 524, 510, 523, 543, 517, 493, 498, 517, 501, 512, 504, 488, 496, 492, 504, 490, 482, 471, 501, 507, 510, 519, 493, 489, 520, 519, 514, 536, 532, 511, 480, 526, 498, 536, 495, 507, 543, 522, 511, 506, 513, 541, 518, 528, 504, 556, 516, 532, 468, 554] payload_rows_per_dst=[5568, 5590, 5526, 5386, 5516, 5648, 5584, 5602] dup_rows_per_dst=[2690, 2616, 2682, 2468, 2600, 2618, 2692, 2750] combine_rows_per_src=[5568, 5590, 5526, 5386, 5516, 5648, 5584, 5602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229976 combine_payload_bytes=279838720 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280068696 ingress_bytes=610140160 link_bytes=610140160 token_bytes=559907416 weight_movement_bytes=330301440 total_nvlink_bytes=890208856 weight_to_token_ratio=0.589922 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061134089e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44632 payload_rows_remote=39174 dup_rows=20904 dup_rows_remote=18352 dedup_ratio=0.318970 router_tokens_per_expert=[516, 529, 530, 532, 484, 542, 483, 508, 555, 507, 515, 538, 500, 514, 506, 502, 510, 482, 538, 505, 491, 518, 526, 504, 541, 530, 536, 514, 541, 506, 484, 478, 532, 519, 520, 518, 551, 499, 510, 512, 500, 460, 513, 523, 474, 503, 511, 521, 496, 495, 549, 494, 473, 500, 501, 497, 524, 550, 506, 531, 502, 512, 479, 528] payload_rows_per_dst=[5550, 5614, 5566, 5656, 5644, 5514, 5458, 5630] dup_rows_per_dst=[2698, 2660, 2582, 2604, 2678, 2496, 2552, 2634] combine_rows_per_src=[5550, 5614, 5566, 5656, 5644, 5514, 5458, 5630] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=229112 combine_payload_bytes=279523328 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=279752440 ingress_bytes=719925248 link_bytes=719925248 token_bytes=559275768 weight_movement_bytes=440401920 total_nvlink_bytes=999677688 weight_to_token_ratio=0.787450 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252068414e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=228776 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278203816 ingress_bytes=388075520 link_bytes=388075520 token_bytes=556178856 weight_movement_bytes=110100480 total_nvlink_bytes=666279336 weight_to_token_ratio=0.197959 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749271567e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=229376 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278706176 ingress_bytes=388577280 link_bytes=388577280 token_bytes=557182976 weight_movement_bytes=110100480 total_nvlink_bytes=667283456 weight_to_token_ratio=0.197602 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757997999e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280799232 dispatch_meta_bytes=230104 combine_payload_bytes=280799232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281029336 ingress_bytes=390899712 link_bytes=390899712 token_bytes=561828568 weight_movement_bytes=110100480 total_nvlink_bytes=671929048 weight_to_token_ratio=0.195968 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.798388911e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38772 dup_rows=21116 dup_rows_remote=18498 dedup_ratio=0.322205 router_tokens_per_expert=[510, 512, 530, 514, 497, 526, 528, 512, 509, 515, 502, 494, 534, 515, 524, 510, 523, 543, 517, 493, 498, 517, 501, 512, 504, 488, 496, 492, 504, 490, 482, 471, 501, 507, 510, 519, 493, 489, 520, 519, 514, 536, 532, 511, 480, 526, 498, 536, 495, 507, 543, 522, 511, 506, 513, 541, 518, 528, 504, 556, 516, 532, 468, 554] payload_rows_per_dst=[5568, 5590, 5526, 5386, 5516, 5648, 5584, 5602] dup_rows_per_dst=[2690, 2616, 2682, 2468, 2600, 2618, 2692, 2750] combine_rows_per_src=[5568, 5590, 5526, 5386, 5516, 5648, 5584, 5602] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38938 dup_rows=21068 dup_rows_remote=18380 dedup_ratio=0.321472 router_tokens_per_expert=[477, 534, 503, 494, 495, 547, 511, 527, 490, 486, 511, 473, 571, 506, 504, 523, 531, 496, 506, 557, 519, 479, 521, 490, 509, 500, 517, 520, 507, 492, 511, 537, 518, 505, 474, 507, 494, 510, 545, 498, 505, 525, 552, 529, 522, 524, 462, 532, 484, 511, 543, 520, 495, 511, 523, 527, 532, 542, 495, 485, 516, 524, 515, 499] payload_rows_per_dst=[5444, 5404, 5608, 5528, 5540, 5730, 5684, 5530] dup_rows_per_dst=[2502, 2420, 2590, 2658, 2560, 2802, 2848, 2688] combine_rows_per_src=[5444, 5404, 5608, 5528, 5540, 5730, 5684, 5530] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38768 dup_rows=21168 dup_rows_remote=18370 dedup_ratio=0.322998 router_tokens_per_expert=[504, 508, 526, 511, 521, 512, 504, 497, 498, 472, 503, 526, 525, 509, 494, 548, 522, 498, 498, 466, 521, 480, 496, 503, 560, 544, 531, 503, 502, 526, 520, 513, 513, 503, 499, 475, 517, 530, 513, 519, 485, 459, 583, 539, 503, 528, 542, 490, 491, 512, 461, 502, 498, 511, 493, 541, 547, 513, 517, 523, 555, 503, 553, 509] payload_rows_per_dst=[5514, 5514, 5454, 5600, 5504, 5632, 5496, 5654] dup_rows_per_dst=[2652, 2636, 2514, 2798, 2634, 2626, 2522, 2786] combine_rows_per_src=[5514, 5514, 5454, 5600, 5504, 5632, 5496, 5654] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38902 dup_rows=21120 dup_rows_remote=18408 dedup_ratio=0.322266 router_tokens_per_expert=[508, 486, 513, 502, 519, 536, 497, 522, 488, 512, 515, 513, 514, 523, 547, 496, 510, 523, 540, 502, 504, 493, 536, 519, 570, 525, 485, 504, 515, 490, 529, 461, 494, 516, 521, 518, 560, 538, 522, 532, 531, 501, 525, 527, 530, 547, 546, 473, 509, 539, 507, 447, 483, 469, 481, 554, 469, 507, 514, 522, 505, 475, 520, 489] payload_rows_per_dst=[5514, 5556, 5804, 5580, 5672, 5382, 5460, 5448] dup_rows_per_dst=[2712, 2664, 3056, 2588, 2716, 2372, 2518, 2494] combine_rows_per_src=[5514, 5556, 5804, 5580, 5672, 5382, 5460, 5448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=229080 combine_payload_bytes=277917696 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278146776 ingress_bytes=498118656 link_bytes=498118656 token_bytes=556064472 weight_movement_bytes=220200960 total_nvlink_bytes=776265432 weight_to_token_ratio=0.395999 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.663102692e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280799232 dispatch_meta_bytes=230104 combine_payload_bytes=280799232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281029336 ingress_bytes=390899712 link_bytes=390899712 token_bytes=561828568 weight_movement_bytes=110100480 total_nvlink_bytes=671929048 weight_to_token_ratio=0.195968 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.798388911e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38966 dup_rows=21070 dup_rows_remote=18540 dedup_ratio=0.321503 router_tokens_per_expert=[509, 532, 496, 526, 485, 503, 522, 504, 512, 526, 544, 526, 537, 521, 498, 568, 516, 484, 490, 484, 481, 539, 520, 501, 529, 495, 494, 507, 555, 500, 502, 505, 514, 478, 489, 487, 486, 528, 534, 486, 508, 546, 534, 508, 522, 500, 477, 522, 535, 535, 501, 529, 467, 519, 564, 524, 468, 498, 503, 527, 509, 544, 495, 520] payload_rows_per_dst=[5594, 5642, 5500, 5572, 5480, 5598, 5576, 5504] dup_rows_per_dst=[2560, 2822, 2530, 2602, 2524, 2636, 2772, 2624] combine_rows_per_src=[5594, 5642, 5500, 5572, 5480, 5598, 5576, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38902 dup_rows=21120 dup_rows_remote=18408 dedup_ratio=0.322266 router_tokens_per_expert=[508, 486, 513, 502, 519, 536, 497, 522, 488, 512, 515, 513, 514, 523, 547, 496, 510, 523, 540, 502, 504, 493, 536, 519, 570, 525, 485, 504, 515, 490, 529, 461, 494, 516, 521, 518, 560, 538, 522, 532, 531, 501, 525, 527, 530, 547, 546, 473, 509, 539, 507, 447, 483, 469, 481, 554, 469, 507, 514, 522, 505, 475, 520, 489] payload_rows_per_dst=[5514, 5556, 5804, 5580, 5672, 5382, 5460, 5448] dup_rows_per_dst=[2712, 2664, 3056, 2588, 2716, 2372, 2518, 2494] combine_rows_per_src=[5514, 5556, 5804, 5580, 5672, 5382, 5460, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=229080 combine_payload_bytes=277917696 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278146776 ingress_bytes=498118656 link_bytes=498118656 token_bytes=556064472 weight_movement_bytes=220200960 total_nvlink_bytes=776265432 weight_to_token_ratio=0.395999 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.663102692e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=39016 dup_rows=21042 dup_rows_remote=18396 dedup_ratio=0.321075 router_tokens_per_expert=[517, 503, 507, 504, 535, 514, 485, 510, 493, 505, 525, 542, 516, 503, 482, 496, 523, 489, 501, 499, 512, 497, 494, 535, 546, 519, 537, 505, 522, 520, 496, 506, 514, 519, 518, 525, 518, 508, 559, 552, 562, 550, 483, 511, 536, 518, 483, 476, 550, 490, 517, 522, 509, 495, 503, 495, 497, 493, 484, 532, 478, 517, 525, 491] payload_rows_per_dst=[5522, 5478, 5492, 5626, 5688, 5608, 5576, 5504] dup_rows_per_dst=[2628, 2646, 2608, 2676, 2738, 2630, 2586, 2530] combine_rows_per_src=[5522, 5478, 5492, 5626, 5688, 5608, 5576, 5504] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229272 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279336856 ingress_bytes=499308544 link_bytes=499308544 token_bytes=558444440 weight_movement_bytes=220200960 total_nvlink_bytes=778645400 weight_to_token_ratio=0.394311 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683796802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277889024 dispatch_meta_bytes=228552 combine_payload_bytes=277889024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278117576 ingress_bytes=387989504 link_bytes=387989504 token_bytes=556006600 weight_movement_bytes=110100480 total_nvlink_bytes=666107080 weight_to_token_ratio=0.198020 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747775608e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38966 dup_rows=21070 dup_rows_remote=18540 dedup_ratio=0.321503 router_tokens_per_expert=[509, 532, 496, 526, 485, 503, 522, 504, 512, 526, 544, 526, 537, 521, 498, 568, 516, 484, 490, 484, 481, 539, 520, 501, 529, 495, 494, 507, 555, 500, 502, 505, 514, 478, 489, 487, 486, 528, 534, 486, 508, 546, 534, 508, 522, 500, 477, 522, 535, 535, 501, 529, 467, 519, 564, 524, 468, 498, 503, 527, 509, 544, 495, 520] payload_rows_per_dst=[5594, 5642, 5500, 5572, 5480, 5598, 5576, 5504] dup_rows_per_dst=[2560, 2822, 2530, 2602, 2524, 2636, 2772, 2624] combine_rows_per_src=[5594, 5642, 5500, 5572, 5480, 5598, 5576, 5504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38852 dup_rows=21108 dup_rows_remote=18460 dedup_ratio=0.322083 router_tokens_per_expert=[520, 513, 518, 519, 488, 528, 480, 524, 525, 526, 542, 473, 499, 482, 545, 520, 525, 502, 507, 501, 518, 540, 488, 493, 482, 511, 473, 497, 529, 483, 540, 508, 478, 501, 498, 504, 519, 510, 529, 536, 548, 502, 497, 540, 522, 524, 517, 480, 498, 503, 494, 500, 512, 523, 501, 532, 535, 564, 484, 513, 542, 532, 512, 519] payload_rows_per_dst=[5542, 5576, 5548, 5522, 5538, 5578, 5472, 5652] dup_rows_per_dst=[2638, 2648, 2600, 2524, 2612, 2682, 2654, 2750] combine_rows_per_src=[5542, 5576, 5548, 5522, 5538, 5578, 5472, 5652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229240 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279078776 ingress_bytes=499050496 link_bytes=499050496 token_bytes=557928312 weight_movement_bytes=220200960 total_nvlink_bytes=778129272 weight_to_token_ratio=0.394676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679308922e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38948 dup_rows=21052 dup_rows_remote=18394 dedup_ratio=0.321228 router_tokens_per_expert=[499, 506, 529, 526, 507, 519, 536, 474, 502, 482, 484, 518, 501, 539, 510, 536, 499, 554, 527, 498, 484, 536, 508, 486, 538, 532, 474, 507, 522, 528, 493, 517, 528, 512, 476, 510, 516, 518, 522, 515, 535, 529, 507, 519, 538, 501, 546, 518, 509, 533, 498, 525, 499, 477, 554, 516, 488, 495, 482, 502, 499, 547, 465, 518] payload_rows_per_dst=[5600, 5480, 5534, 5524, 5536, 5660, 5650, 5500] dup_rows_per_dst=[2592, 2664, 2650, 2698, 2658, 2726, 2572, 2492] combine_rows_per_src=[5600, 5480, 5534, 5524, 5536, 5660, 5650, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44554 payload_rows_remote=39040 dup_rows=20982 dup_rows_remote=18454 dedup_ratio=0.320160 router_tokens_per_expert=[504, 517, 534, 508, 536, 525, 536, 534, 568, 512, 505, 516, 492, 526, 509, 509, 503, 477, 481, 533, 503, 477, 523, 515, 469, 506, 535, 507, 524, 532, 478, 546, 497, 478, 526, 522, 492, 492, 511, 503, 536, 521, 541, 528, 479, 504, 485, 528, 484, 525, 528, 523, 494, 502, 536, 514, 508, 497, 513, 502, 511, 494, 539, 515] payload_rows_per_dst=[5620, 5620, 5482, 5598, 5514, 5582, 5588, 5550] dup_rows_per_dst=[2768, 2654, 2542, 2596, 2528, 2662, 2624, 2608] combine_rows_per_src=[5620, 5620, 5482, 5598, 5514, 5582, 5588, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38946 dup_rows=21002 dup_rows_remote=18434 dedup_ratio=0.320465 router_tokens_per_expert=[538, 514, 510, 521, 529, 531, 495, 521, 528, 512, 471, 513, 511, 481, 531, 478, 479, 539, 486, 521, 477, 499, 506, 520, 486, 499, 550, 518, 539, 515, 519, 535, 486, 498, 499, 547, 493, 517, 523, 487, 502, 540, 486, 503, 563, 519, 491, 507, 482, 534, 549, 488, 514, 534, 471, 506, 534, 525, 507, 560, 522, 480, 504, 525] payload_rows_per_dst=[5580, 5522, 5498, 5610, 5500, 5598, 5588, 5638] dup_rows_per_dst=[2738, 2528, 2556, 2712, 2600, 2624, 2568, 2676] combine_rows_per_src=[5580, 5522, 5498, 5610, 5500, 5598, 5588, 5638] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=230024 combine_payload_bytes=279308288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279538312 ingress_bytes=389408768 link_bytes=389408768 token_bytes=558846600 weight_movement_bytes=110100480 total_nvlink_bytes=668947080 weight_to_token_ratio=0.197014 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772458943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=229248 combine_payload_bytes=278491136 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278720384 ingress_bytes=498692096 link_bytes=498692096 token_bytes=557211520 weight_movement_bytes=220200960 total_nvlink_bytes=777412480 weight_to_token_ratio=0.395184 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673075757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229240 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279078776 ingress_bytes=499050496 link_bytes=499050496 token_bytes=557928312 weight_movement_bytes=220200960 total_nvlink_bytes=778129272 weight_to_token_ratio=0.394676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679308922e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=229648 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279896336 ingress_bytes=499867648 link_bytes=499867648 token_bytes=559563024 weight_movement_bytes=220200960 total_nvlink_bytes=779763984 weight_to_token_ratio=0.393523 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693520540e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229976 combine_payload_bytes=279838720 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280068696 ingress_bytes=610140160 link_bytes=610140160 token_bytes=559907416 weight_movement_bytes=330301440 total_nvlink_bytes=890208856 weight_to_token_ratio=0.589922 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061134089e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229520 combine_payload_bytes=279164928 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279394448 ingress_bytes=389265408 link_bytes=389265408 token_bytes=558559376 weight_movement_bytes=110100480 total_nvlink_bytes=668659856 weight_to_token_ratio=0.197115 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769965677e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44654 payload_rows_remote=39028 dup_rows=20882 dup_rows_remote=18196 dedup_ratio=0.318634 router_tokens_per_expert=[516, 521, 490, 515, 500, 510, 515, 513, 517, 492, 507, 534, 511, 504, 497, 510, 508, 476, 518, 470, 469, 487, 500, 523, 528, 536, 533, 547, 517, 508, 518, 518, 525, 522, 498, 488, 533, 479, 524, 500, 549, 529, 550, 540, 475, 490, 508, 515, 533, 508, 488, 504, 514, 522, 492, 486, 533, 506, 514, 494, 530, 525, 541, 545] payload_rows_per_dst=[5578, 5562, 5514, 5672, 5574, 5626, 5484, 5644] dup_rows_per_dst=[2582, 2582, 2388, 2738, 2564, 2686, 2610, 2732] combine_rows_per_src=[5578, 5562, 5514, 5672, 5574, 5626, 5484, 5644] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38946 dup_rows=21002 dup_rows_remote=18434 dedup_ratio=0.320465 router_tokens_per_expert=[538, 514, 510, 521, 529, 531, 495, 521, 528, 512, 471, 513, 511, 481, 531, 478, 479, 539, 486, 521, 477, 499, 506, 520, 486, 499, 550, 518, 539, 515, 519, 535, 486, 498, 499, 547, 493, 517, 523, 487, 502, 540, 486, 503, 563, 519, 491, 507, 482, 534, 549, 488, 514, 534, 471, 506, 534, 525, 507, 560, 522, 480, 504, 525] payload_rows_per_dst=[5580, 5522, 5498, 5610, 5500, 5598, 5588, 5638] dup_rows_per_dst=[2738, 2528, 2556, 2712, 2600, 2624, 2568, 2676] combine_rows_per_src=[5580, 5522, 5498, 5610, 5500, 5598, 5588, 5638] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279179264 dispatch_meta_bytes=229368 combine_payload_bytes=279179264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279408632 ingress_bytes=389279744 link_bytes=389279744 token_bytes=558587896 weight_movement_bytes=110100480 total_nvlink_bytes=668688376 weight_to_token_ratio=0.197105 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770215003e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38764 dup_rows=21150 dup_rows_remote=18308 dedup_ratio=0.322723 router_tokens_per_expert=[537, 503, 501, 493, 494, 532, 510, 489, 531, 505, 542, 497, 543, 549, 540, 532, 505, 507, 508, 526, 500, 520, 495, 478, 499, 508, 510, 529, 476, 510, 464, 519, 473, 520, 486, 520, 530, 502, 521, 543, 502, 522, 538, 504, 488, 514, 498, 502, 516, 476, 512, 507, 492, 553, 517, 525, 481, 515, 498, 529, 524, 539, 534, 535] payload_rows_per_dst=[5178, 5688, 5828, 5400, 5604, 5532, 5534, 5622] dup_rows_per_dst=[2016, 2790, 3174, 2476, 2586, 2604, 2662, 2842] combine_rows_per_src=[5178, 5688, 5828, 5400, 5604, 5532, 5534, 5622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44648 payload_rows_remote=38976 dup_rows=20888 dup_rows_remote=18222 dedup_ratio=0.318726 router_tokens_per_expert=[514, 521, 511, 519, 510, 512, 556, 520, 497, 499, 526, 510, 551, 504, 479, 539, 521, 460, 518, 529, 518, 503, 545, 544, 493, 526, 539, 511, 543, 510, 500, 547, 502, 510, 524, 515, 529, 483, 527, 525, 515, 498, 474, 505, 523, 505, 492, 501, 525, 514, 491, 488, 519, 521, 511, 477, 488, 491, 485, 492, 515, 515, 523, 510] payload_rows_per_dst=[5598, 5556, 5608, 5672, 5572, 5536, 5610, 5496] dup_rows_per_dst=[2728, 2654, 2668, 2666, 2658, 2490, 2482, 2542] combine_rows_per_src=[5598, 5556, 5608, 5672, 5572, 5536, 5610, 5496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44328 payload_rows_remote=38508 dup_rows=21208 dup_rows_remote=18098 dedup_ratio=0.323608 router_tokens_per_expert=[515, 515, 528, 502, 529, 530, 523, 548, 542, 543, 485, 494, 499, 505, 539, 566, 502, 506, 488, 499, 485, 494, 544, 484, 514, 494, 502, 511, 472, 512, 533, 549, 547, 500, 521, 501, 522, 515, 518, 527, 491, 486, 494, 526, 488, 505, 513, 535, 495, 517, 523, 498, 539, 549, 494, 519, 496, 533, 516, 532, 479, 501, 478, 458] payload_rows_per_dst=[5820, 5460, 5494, 5614, 5586, 5544, 5554, 5256] dup_rows_per_dst=[3110, 2626, 2510, 2624, 2734, 2514, 2650, 2440] combine_rows_per_src=[5820, 5460, 5494, 5614, 5586, 5544, 5554, 5256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5578452032 prefetch_bytes=990904320 reduce_bytes=1321205760 weight_movement_bytes=2312110080 egress_bytes=2790372416 ingress_bytes=5100189696 link_bytes=5100189696 total_nvlink_bytes=7890562112 weight_to_token_ratio=0.414472 prefetch_share_of_ingress=0.194288 dedup_ratio=0.321655 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=12 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.870068718e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=228896 combine_payload_bytes=279752704 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279981600 ingress_bytes=389853184 link_bytes=389853184 token_bytes=559734304 weight_movement_bytes=110100480 total_nvlink_bytes=669834784 weight_to_token_ratio=0.196701 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780188068e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229520 combine_payload_bytes=279164928 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279394448 ingress_bytes=389265408 link_bytes=389265408 token_bytes=558559376 weight_movement_bytes=110100480 total_nvlink_bytes=668659856 weight_to_token_ratio=0.197115 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769965677e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5592390304 prefetch_bytes=990904320 reduce_bytes=220200960 weight_movement_bytes=1211105280 egress_bytes=2797343392 ingress_bytes=4006152192 link_bytes=4006152192 total_nvlink_bytes=6803495584 weight_to_token_ratio=0.216563 prefetch_share_of_ingress=0.247346 dedup_ratio=0.320859 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=2 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=6.967357561e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44328 payload_rows_remote=38508 dup_rows=21208 dup_rows_remote=18098 dedup_ratio=0.323608 router_tokens_per_expert=[515, 515, 528, 502, 529, 530, 523, 548, 542, 543, 485, 494, 499, 505, 539, 566, 502, 506, 488, 499, 485, 494, 544, 484, 514, 494, 502, 511, 472, 512, 533, 549, 547, 500, 521, 501, 522, 515, 518, 527, 491, 486, 494, 526, 488, 505, 513, 535, 495, 517, 523, 498, 539, 549, 494, 519, 496, 533, 516, 532, 479, 501, 478, 458] payload_rows_per_dst=[5820, 5460, 5494, 5614, 5586, 5544, 5554, 5256] dup_rows_per_dst=[3110, 2626, 2510, 2624, 2734, 2514, 2650, 2440] combine_rows_per_src=[5820, 5460, 5494, 5614, 5586, 5544, 5554, 5256] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5576587304 prefetch_bytes=880803840 reduce_bytes=990904320 weight_movement_bytes=1871708160 egress_bytes=2789439528 ingress_bytes=4658855936 link_bytes=4658855936 total_nvlink_bytes=7448295464 weight_to_token_ratio=0.335637 prefetch_share_of_ingress=0.189060 dedup_ratio=0.321408 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=9 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.102516723e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=39094 dup_rows=21036 dup_rows_remote=18476 dedup_ratio=0.320984 router_tokens_per_expert=[505, 499, 516, 526, 492, 514, 557, 509, 497, 495, 498, 500, 522, 553, 524, 511, 516, 519, 450, 470, 550, 476, 499, 503, 528, 491, 530, 478, 537, 480, 545, 453, 508, 495, 542, 514, 487, 519, 523, 519, 503, 524, 494, 537, 504, 500, 507, 499, 520, 497, 577, 511, 561, 535, 494, 528, 530, 507, 516, 515, 513, 523, 493, 530] payload_rows_per_dst=[5550, 5510, 5406, 5586, 5586, 5638, 5656, 5568] dup_rows_per_dst=[2686, 2690, 2560, 2498, 2628, 2498, 2790, 2686] combine_rows_per_src=[5550, 5510, 5406, 5586, 5586, 5638, 5656, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=228896 combine_payload_bytes=279752704 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279981600 ingress_bytes=389853184 link_bytes=389853184 token_bytes=559734304 weight_movement_bytes=110100480 total_nvlink_bytes=669834784 weight_to_token_ratio=0.196701 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780188068e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277860352 dispatch_meta_bytes=228288 combine_payload_bytes=277860352 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278088640 ingress_bytes=498061312 link_bytes=498061312 token_bytes=555948992 weight_movement_bytes=220200960 total_nvlink_bytes=776149952 weight_to_token_ratio=0.396081 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662105386e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38852 dup_rows=21108 dup_rows_remote=18460 dedup_ratio=0.322083 router_tokens_per_expert=[520, 513, 518, 519, 488, 528, 480, 524, 525, 526, 542, 473, 499, 482, 545, 520, 525, 502, 507, 501, 518, 540, 488, 493, 482, 511, 473, 497, 529, 483, 540, 508, 478, 501, 498, 504, 519, 510, 529, 536, 548, 502, 497, 540, 522, 524, 517, 480, 498, 503, 494, 500, 512, 523, 501, 532, 535, 564, 484, 513, 542, 532, 512, 519] payload_rows_per_dst=[5542, 5576, 5548, 5522, 5538, 5578, 5472, 5652] dup_rows_per_dst=[2638, 2648, 2600, 2524, 2612, 2682, 2654, 2750] combine_rows_per_src=[5542, 5576, 5548, 5522, 5538, 5578, 5472, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279379968 dispatch_meta_bytes=228792 combine_payload_bytes=279379968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279608760 ingress_bytes=499580928 link_bytes=499580928 token_bytes=558988728 weight_movement_bytes=220200960 total_nvlink_bytes=779189688 weight_to_token_ratio=0.393927 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688534007e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38948 dup_rows=21052 dup_rows_remote=18394 dedup_ratio=0.321228 router_tokens_per_expert=[499, 506, 529, 526, 507, 519, 536, 474, 502, 482, 484, 518, 501, 539, 510, 536, 499, 554, 527, 498, 484, 536, 508, 486, 538, 532, 474, 507, 522, 528, 493, 517, 528, 512, 476, 510, 516, 518, 522, 515, 535, 529, 507, 519, 538, 501, 546, 518, 509, 533, 498, 525, 499, 477, 554, 516, 488, 495, 482, 502, 499, 547, 465, 518] payload_rows_per_dst=[5600, 5480, 5534, 5524, 5536, 5660, 5650, 5500] dup_rows_per_dst=[2592, 2664, 2650, 2698, 2658, 2726, 2572, 2492] combine_rows_per_src=[5600, 5480, 5534, 5524, 5536, 5660, 5650, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=276025344 dispatch_meta_bytes=226424 combine_payload_bytes=276025344 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=276251768 ingress_bytes=496226304 link_bytes=496226304 token_bytes=552277112 weight_movement_bytes=220200960 total_nvlink_bytes=772478072 weight_to_token_ratio=0.398715 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.630191579e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582435984 prefetch_bytes=880803840 reduce_bytes=990904320 weight_movement_bytes=1871708160 egress_bytes=2792363664 ingress_bytes=4661780480 link_bytes=4661780480 total_nvlink_bytes=7454144144 weight_to_token_ratio=0.335285 prefetch_share_of_ingress=0.188942 dedup_ratio=0.320667 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=9 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.107602986e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=39094 dup_rows=21036 dup_rows_remote=18476 dedup_ratio=0.320984 router_tokens_per_expert=[505, 499, 516, 526, 492, 514, 557, 509, 497, 495, 498, 500, 522, 553, 524, 511, 516, 519, 450, 470, 550, 476, 499, 503, 528, 491, 530, 478, 537, 480, 545, 453, 508, 495, 542, 514, 487, 519, 523, 519, 503, 524, 494, 537, 504, 500, 507, 499, 520, 497, 577, 511, 561, 535, 494, 528, 530, 507, 516, 515, 513, 523, 493, 530] payload_rows_per_dst=[5550, 5510, 5406, 5586, 5586, 5638, 5656, 5568] dup_rows_per_dst=[2686, 2690, 2560, 2498, 2628, 2498, 2790, 2686] combine_rows_per_src=[5550, 5510, 5406, 5586, 5586, 5638, 5656, 5568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5592390304 prefetch_bytes=990904320 reduce_bytes=220200960 weight_movement_bytes=1211105280 egress_bytes=2797343392 ingress_bytes=4006152192 link_bytes=4006152192 total_nvlink_bytes=6803495584 weight_to_token_ratio=0.216563 prefetch_share_of_ingress=0.247346 dedup_ratio=0.320859 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=2 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=6.967357561e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5574491456 prefetch_bytes=880803840 reduce_bytes=1321205760 weight_movement_bytes=2202009600 egress_bytes=2788390208 ingress_bytes=4988110848 link_bytes=4988110848 total_nvlink_bytes=7776501056 weight_to_token_ratio=0.395015 prefetch_share_of_ingress=0.176581 dedup_ratio=0.321558 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=12 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.675145168e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=230280 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280456072 ingress_bytes=390326272 link_bytes=390326272 token_bytes=560681864 weight_movement_bytes=110100480 total_nvlink_bytes=670782344 weight_to_token_ratio=0.196369 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788415846e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=229248 combine_payload_bytes=278491136 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278720384 ingress_bytes=498692096 link_bytes=498692096 token_bytes=557211520 weight_movement_bytes=220200960 total_nvlink_bytes=777412480 weight_to_token_ratio=0.395184 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673075757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5575297088 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2788794432 ingress_bytes=4768311296 link_bytes=4768311296 total_nvlink_bytes=7557105728 weight_to_token_ratio=0.355462 prefetch_share_of_ingress=0.207810 dedup_ratio=0.321536 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.292877596e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5562444616 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2782364488 ingress_bytes=4761888768 link_bytes=4761888768 total_nvlink_bytes=7544253256 weight_to_token_ratio=0.356284 prefetch_share_of_ingress=0.208091 dedup_ratio=0.322052 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.281707764e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=230280 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280456072 ingress_bytes=390326272 link_bytes=390326272 token_bytes=560681864 weight_movement_bytes=110100480 total_nvlink_bytes=670782344 weight_to_token_ratio=0.196369 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788415846e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279179264 dispatch_meta_bytes=229368 combine_payload_bytes=279179264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279408632 ingress_bytes=389279744 link_bytes=389279744 token_bytes=558587896 weight_movement_bytes=110100480 total_nvlink_bytes=668688376 weight_to_token_ratio=0.197105 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770215003e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582639464 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=2792466792 ingress_bytes=4551780352 link_bytes=4551780352 total_nvlink_bytes=7344247144 weight_to_token_ratio=0.315551 prefetch_share_of_ingress=0.217696 dedup_ratio=0.321127 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.916294671e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none - [2026-08-26 16:58:52] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 365.3 | throughput per GPU (TFLOP/s/GPU): 338.0 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.868143E+00 | loss scale: 1.0 | grad norm: 19.094 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5578452032 prefetch_bytes=990904320 reduce_bytes=1321205760 weight_movement_bytes=2312110080 egress_bytes=2790372416 ingress_bytes=5100189696 link_bytes=5100189696 total_nvlink_bytes=7890562112 weight_to_token_ratio=0.414472 prefetch_share_of_ingress=0.194288 dedup_ratio=0.321655 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=12 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.870068718e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5576587304 prefetch_bytes=880803840 reduce_bytes=990904320 weight_movement_bytes=1871708160 egress_bytes=2789439528 ingress_bytes=4658855936 link_bytes=4658855936 total_nvlink_bytes=7448295464 weight_to_token_ratio=0.335637 prefetch_share_of_ingress=0.189060 dedup_ratio=0.321408 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=9 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.102516723e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none + [2026-08-26 18:10:58] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 363.1 | throughput per GPU (TFLOP/s/GPU): 340.1 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.868757E+00 | loss scale: 1.0 | grad norm: 19.141 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44384 payload_rows_remote=38758 dup_rows=21152 dup_rows_remote=18468 dedup_ratio=0.322754 router_tokens_per_expert=[519, 506, 484, 529, 501, 457, 522, 540, 521, 508, 530, 463, 497, 560, 512, 541, 475, 510, 484, 486, 531, 511, 527, 515, 538, 550, 518, 513, 506, 518, 484, 512, 507, 543, 471, 509, 527, 511, 526, 492, 538, 573, 496, 501, 500, 522, 530, 495, 491, 491, 552, 491, 495, 536, 481, 527, 519, 532, 518, 495, 519, 474, 492, 546] payload_rows_per_dst=[5520, 5636, 5506, 5612, 5460, 5626, 5536, 5488] dup_rows_per_dst=[2596, 2628, 2572, 2666, 2712, 2684, 2592, 2702] combine_rows_per_src=[5520, 5636, 5506, 5612, 5460, 5626, 5536, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44372 payload_rows_remote=38890 dup_rows=21164 dup_rows_remote=18504 dedup_ratio=0.322937 router_tokens_per_expert=[489, 551, 520, 490, 518, 512, 541, 514, 496, 482, 522, 483, 510, 491, 484, 496, 517, 519, 537, 496, 487, 541, 508, 539, 545, 543, 491, 546, 479, 513, 507, 527, 531, 534, 472, 518, 508, 533, 543, 514, 510, 512, 524, 519, 496, 534, 517, 497, 508, 505, 540, 472, 484, 523, 518, 521, 500, 518, 517, 501, 503, 489, 491, 522] payload_rows_per_dst=[5578, 5450, 5626, 5576, 5620, 5564, 5482, 5476] dup_rows_per_dst=[2692, 2478, 2662, 2726, 2686, 2654, 2660, 2606] combine_rows_per_src=[5578, 5450, 5626, 5576, 5620, 5564, 5482, 5476] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44574 payload_rows_remote=38924 dup_rows=20962 dup_rows_remote=18364 dedup_ratio=0.319855 router_tokens_per_expert=[547, 493, 515, 494, 519, 517, 515, 529, 534, 535, 528, 527, 494, 496, 504, 506, 505, 496, 517, 500, 521, 471, 509, 522, 497, 539, 513, 511, 480, 511, 540, 532, 489, 507, 551, 542, 520, 514, 532, 542, 532, 478, 495, 488, 524, 520, 517, 518, 463, 484, 478, 542, 528, 498, 507, 522, 493, 505, 497, 518, 526, 503, 512, 506] payload_rows_per_dst=[5558, 5650, 5530, 5568, 5696, 5532, 5488, 5552] dup_rows_per_dst=[2700, 2598, 2552, 2678, 2698, 2612, 2556, 2568] combine_rows_per_src=[5558, 5650, 5530, 5568, 5696, 5532, 5488, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44578 payload_rows_remote=38886 dup_rows=20958 dup_rows_remote=18256 dedup_ratio=0.319794 router_tokens_per_expert=[501, 487, 528, 517, 517, 483, 504, 510, 525, 555, 503, 513, 492, 529, 516, 493, 505, 525, 478, 513, 514, 533, 493, 530, 532, 517, 522, 505, 498, 524, 553, 546, 527, 519, 499, 485, 523, 515, 511, 518, 515, 484, 490, 508, 514, 470, 524, 524, 490, 544, 501, 525, 498, 461, 519, 490, 486, 559, 550, 503, 539, 500, 525, 491] payload_rows_per_dst=[5504, 5570, 5588, 5692, 5592, 5488, 5522, 5622] dup_rows_per_dst=[2590, 2682, 2594, 2702, 2602, 2570, 2534, 2684] combine_rows_per_src=[5504, 5570, 5588, 5692, 5592, 5488, 5522, 5622] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44404 payload_rows_remote=38874 dup_rows=21132 dup_rows_remote=18600 dedup_ratio=0.322449 router_tokens_per_expert=[528, 491, 491, 528, 512, 501, 540, 541, 526, 544, 524, 504, 550, 519, 510, 482, 526, 496, 507, 484, 514, 459, 515, 511, 493, 519, 481, 529, 518, 536, 524, 505, 510, 488, 482, 524, 513, 515, 500, 499, 513, 523, 512, 513, 491, 509, 509, 516, 508, 518, 509, 511, 496, 567, 518, 532, 506, 499, 495, 509, 509, 517, 514, 535] payload_rows_per_dst=[5602, 5608, 5466, 5572, 5530, 5510, 5574, 5542] dup_rows_per_dst=[2662, 2710, 2558, 2638, 2532, 2662, 2744, 2626] combine_rows_per_src=[5602, 5608, 5466, 5572, 5530, 5510, 5574, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44384 payload_rows_remote=38758 dup_rows=21152 dup_rows_remote=18468 dedup_ratio=0.322754 router_tokens_per_expert=[519, 506, 484, 529, 501, 457, 522, 540, 521, 508, 530, 463, 497, 560, 512, 541, 475, 510, 484, 486, 531, 511, 527, 515, 538, 550, 518, 513, 506, 518, 484, 512, 507, 543, 471, 509, 527, 511, 526, 492, 538, 573, 496, 501, 500, 522, 530, 495, 491, 491, 552, 491, 495, 536, 481, 527, 519, 532, 518, 495, 519, 474, 492, 546] payload_rows_per_dst=[5520, 5636, 5506, 5612, 5460, 5626, 5536, 5488] dup_rows_per_dst=[2596, 2628, 2572, 2666, 2712, 2684, 2592, 2702] combine_rows_per_src=[5520, 5636, 5506, 5612, 5460, 5626, 5536, 5488] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38854 dup_rows=21052 dup_rows_remote=18420 dedup_ratio=0.321228 router_tokens_per_expert=[497, 525, 504, 508, 526, 496, 507, 518, 558, 505, 504, 511, 537, 488, 522, 522, 493, 495, 491, 515, 509, 477, 486, 494, 546, 531, 519, 506, 525, 454, 555, 533, 502, 512, 545, 464, 535, 510, 507, 495, 500, 514, 513, 521, 516, 528, 525, 468, 504, 524, 554, 479, 537, 501, 514, 516, 495, 537, 531, 525, 509, 531, 473, 526] payload_rows_per_dst=[5546, 5654, 5438, 5496, 5556, 5552, 5612, 5630] dup_rows_per_dst=[2616, 2640, 2482, 2544, 2584, 2742, 2812, 2632] combine_rows_per_src=[5546, 5654, 5438, 5496, 5556, 5552, 5612, 5630] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44578 payload_rows_remote=38886 dup_rows=20958 dup_rows_remote=18256 dedup_ratio=0.319794 router_tokens_per_expert=[501, 487, 528, 517, 517, 483, 504, 510, 525, 555, 503, 513, 492, 529, 516, 493, 505, 525, 478, 513, 514, 533, 493, 530, 532, 517, 522, 505, 498, 524, 553, 546, 527, 519, 499, 485, 523, 515, 511, 518, 515, 484, 490, 508, 514, 470, 524, 524, 490, 544, 501, 525, 498, 461, 519, 490, 486, 559, 550, 503, 539, 500, 525, 491] payload_rows_per_dst=[5504, 5570, 5588, 5692, 5592, 5488, 5522, 5622] dup_rows_per_dst=[2590, 2682, 2594, 2702, 2602, 2570, 2534, 2684] combine_rows_per_src=[5504, 5570, 5588, 5692, 5592, 5488, 5522, 5622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44372 payload_rows_remote=38890 dup_rows=21164 dup_rows_remote=18504 dedup_ratio=0.322937 router_tokens_per_expert=[489, 551, 520, 490, 518, 512, 541, 514, 496, 482, 522, 483, 510, 491, 484, 496, 517, 519, 537, 496, 487, 541, 508, 539, 545, 543, 491, 546, 479, 513, 507, 527, 531, 534, 472, 518, 508, 533, 543, 514, 510, 512, 524, 519, 496, 534, 517, 497, 508, 505, 540, 472, 484, 523, 518, 521, 500, 518, 517, 501, 503, 489, 491, 522] payload_rows_per_dst=[5578, 5450, 5626, 5576, 5620, 5564, 5482, 5476] dup_rows_per_dst=[2692, 2478, 2662, 2726, 2686, 2654, 2660, 2606] combine_rows_per_src=[5578, 5450, 5626, 5576, 5620, 5564, 5482, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44700 payload_rows_remote=39042 dup_rows=20836 dup_rows_remote=18328 dedup_ratio=0.317932 router_tokens_per_expert=[513, 524, 546, 494, 503, 545, 526, 476, 463, 509, 493, 520, 490, 502, 518, 533, 524, 524, 505, 531, 497, 516, 484, 502, 527, 522, 517, 494, 505, 528, 544, 503, 559, 501, 520, 489, 538, 502, 465, 486, 510, 485, 504, 510, 524, 522, 483, 532, 548, 549, 498, 507, 512, 505, 503, 521, 524, 526, 484, 505, 504, 534, 505, 535] payload_rows_per_dst=[5542, 5504, 5658, 5642, 5582, 5572, 5610, 5590] dup_rows_per_dst=[2712, 2552, 2508, 2638, 2538, 2568, 2676, 2644] combine_rows_per_src=[5542, 5504, 5658, 5642, 5582, 5572, 5610, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38992 dup_rows=21000 dup_rows_remote=18426 dedup_ratio=0.320435 router_tokens_per_expert=[505, 476, 463, 511, 504, 450, 561, 537, 510, 507, 521, 519, 544, 509, 507, 521, 545, 515, 531, 518, 491, 497, 521, 509, 494, 550, 511, 530, 516, 514, 505, 546, 503, 535, 528, 499, 472, 492, 533, 494, 515, 506, 510, 509, 512, 542, 485, 521, 526, 481, 503, 491, 532, 523, 512, 501, 522, 521, 493, 521, 491, 556, 464, 537] payload_rows_per_dst=[5544, 5568, 5832, 5360, 5512, 5612, 5566, 5542] dup_rows_per_dst=[2574, 2608, 2964, 2426, 2636, 2588, 2572, 2632] combine_rows_per_src=[5544, 5568, 5832, 5360, 5512, 5612, 5566, 5542] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229152 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279236384 ingress_bytes=499208192 link_bytes=499208192 token_bytes=558243616 weight_movement_bytes=220200960 total_nvlink_bytes=778444576 weight_to_token_ratio=0.394453 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682051515e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=229896 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278878728 ingress_bytes=388749312 link_bytes=388749312 token_bytes=557527560 weight_movement_bytes=110100480 total_nvlink_bytes=667628040 weight_to_token_ratio=0.197480 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760989918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44700 payload_rows_remote=39042 dup_rows=20836 dup_rows_remote=18328 dedup_ratio=0.317932 router_tokens_per_expert=[513, 524, 546, 494, 503, 545, 526, 476, 463, 509, 493, 520, 490, 502, 518, 533, 524, 524, 505, 531, 497, 516, 484, 502, 527, 522, 517, 494, 505, 528, 544, 503, 559, 501, 520, 489, 538, 502, 465, 486, 510, 485, 504, 510, 524, 522, 483, 532, 548, 549, 498, 507, 512, 505, 503, 521, 524, 526, 484, 505, 504, 534, 505, 535] payload_rows_per_dst=[5542, 5504, 5658, 5642, 5582, 5572, 5610, 5590] dup_rows_per_dst=[2712, 2552, 2508, 2638, 2538, 2568, 2676, 2644] combine_rows_per_src=[5542, 5504, 5658, 5642, 5582, 5572, 5610, 5590] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277817344 dispatch_meta_bytes=228904 combine_payload_bytes=277817344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278046248 ingress_bytes=387917824 link_bytes=387917824 token_bytes=555863592 weight_movement_bytes=110100480 total_nvlink_bytes=665964072 weight_to_token_ratio=0.198071 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746528975e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278505472 dispatch_meta_bytes=229096 combine_payload_bytes=278505472 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278734568 ingress_bytes=498706432 link_bytes=498706432 token_bytes=557240040 weight_movement_bytes=220200960 total_nvlink_bytes=777441000 weight_to_token_ratio=0.395164 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673325084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38992 dup_rows=21000 dup_rows_remote=18426 dedup_ratio=0.320435 router_tokens_per_expert=[505, 476, 463, 511, 504, 450, 561, 537, 510, 507, 521, 519, 544, 509, 507, 521, 545, 515, 531, 518, 491, 497, 521, 509, 494, 550, 511, 530, 516, 514, 505, 546, 503, 535, 528, 499, 472, 492, 533, 494, 515, 506, 510, 509, 512, 542, 485, 521, 526, 481, 503, 491, 532, 523, 512, 501, 522, 521, 493, 521, 491, 556, 464, 537] payload_rows_per_dst=[5544, 5568, 5832, 5360, 5512, 5612, 5566, 5542] dup_rows_per_dst=[2574, 2608, 2964, 2426, 2636, 2588, 2572, 2632] combine_rows_per_src=[5544, 5568, 5832, 5360, 5512, 5612, 5566, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=228568 combine_payload_bytes=278734848 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=278963416 ingress_bytes=829237248 link_bytes=829237248 token_bytes=557698264 weight_movement_bytes=550502400 total_nvlink_bytes=1108200664 weight_to_token_ratio=0.987097 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.442179960e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229576 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278993096 ingress_bytes=388864000 link_bytes=388864000 token_bytes=557756616 weight_movement_bytes=110100480 total_nvlink_bytes=667857096 weight_to_token_ratio=0.197399 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762984531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229152 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279236384 ingress_bytes=499208192 link_bytes=499208192 token_bytes=558243616 weight_movement_bytes=220200960 total_nvlink_bytes=778444576 weight_to_token_ratio=0.394453 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682051515e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=228568 combine_payload_bytes=278734848 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=278963416 ingress_bytes=829237248 link_bytes=829237248 token_bytes=557698264 weight_movement_bytes=550502400 total_nvlink_bytes=1108200664 weight_to_token_ratio=0.987097 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.442179960e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=229896 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278878728 ingress_bytes=388749312 link_bytes=388749312 token_bytes=557527560 weight_movement_bytes=110100480 total_nvlink_bytes=667628040 weight_to_token_ratio=0.197480 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760989918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278505472 dispatch_meta_bytes=229096 combine_payload_bytes=278505472 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278734568 ingress_bytes=498706432 link_bytes=498706432 token_bytes=557240040 weight_movement_bytes=220200960 total_nvlink_bytes=777441000 weight_to_token_ratio=0.395164 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673325084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279853056 dispatch_meta_bytes=229480 combine_payload_bytes=279853056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280082536 ingress_bytes=389953536 link_bytes=389953536 token_bytes=559935592 weight_movement_bytes=110100480 total_nvlink_bytes=670036072 weight_to_token_ratio=0.196631 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781933354e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279494656 dispatch_meta_bytes=229672 combine_payload_bytes=279494656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279724328 ingress_bytes=389595136 link_bytes=389595136 token_bytes=559218984 weight_movement_bytes=110100480 total_nvlink_bytes=669319464 weight_to_token_ratio=0.196883 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775700189e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38964 dup_rows=20976 dup_rows_remote=18366 dedup_ratio=0.320068 router_tokens_per_expert=[507, 562, 519, 555, 538, 494, 513, 526, 516, 535, 494, 485, 521, 462, 527, 516, 510, 504, 505, 504, 515, 525, 548, 536, 522, 497, 517, 493, 489, 537, 503, 532, 550, 535, 510, 490, 554, 520, 502, 511, 503, 534, 529, 523, 512, 496, 490, 516, 506, 528, 494, 495, 517, 483, 483, 487, 491, 488, 498, 522, 472, 478, 512, 532] payload_rows_per_dst=[5598, 5574, 5610, 5554, 5656, 5596, 5504, 5468] dup_rows_per_dst=[2830, 2538, 2684, 2626, 2688, 2610, 2482, 2518] combine_rows_per_src=[5598, 5574, 5610, 5554, 5656, 5596, 5504, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38914 dup_rows=21108 dup_rows_remote=18476 dedup_ratio=0.322083 router_tokens_per_expert=[524, 493, 513, 498, 533, 483, 499, 501, 531, 518, 500, 521, 509, 499, 492, 470, 514, 518, 526, 509, 512, 518, 511, 499, 498, 495, 504, 529, 530, 494, 531, 511, 488, 559, 496, 527, 542, 524, 530, 511, 508, 535, 532, 505, 515, 507, 519, 490, 520, 533, 503, 522, 515, 468, 484, 528, 516, 539, 516, 506, 504, 517, 525, 501] payload_rows_per_dst=[5478, 5482, 5586, 5600, 5600, 5584, 5514, 5584] dup_rows_per_dst=[2610, 2598, 2628, 2584, 2754, 2638, 2632, 2664] combine_rows_per_src=[5478, 5482, 5586, 5600, 5600, 5584, 5514, 5584] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38902 dup_rows=21124 dup_rows_remote=18574 dedup_ratio=0.322327 router_tokens_per_expert=[510, 510, 520, 501, 514, 497, 501, 489, 490, 478, 527, 483, 544, 515, 511, 482, 545, 494, 533, 539, 502, 543, 530, 503, 514, 535, 517, 511, 533, 520, 486, 531, 500, 475, 527, 501, 507, 507, 526, 487, 491, 533, 510, 515, 562, 527, 536, 497, 482, 520, 521, 524, 492, 531, 516, 511, 505, 530, 506, 536, 489, 530, 508, 458] payload_rows_per_dst=[5526, 5510, 5592, 5598, 5458, 5654, 5546, 5528] dup_rows_per_dst=[2558, 2550, 2786, 2696, 2602, 2688, 2648, 2596] combine_rows_per_src=[5526, 5510, 5592, 5598, 5458, 5654, 5546, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38888 dup_rows=20976 dup_rows_remote=18288 dedup_ratio=0.320068 router_tokens_per_expert=[517, 515, 496, 509, 529, 550, 527, 507, 479, 545, 489, 519, 509, 525, 500, 501, 510, 530, 506, 510, 481, 508, 521, 500, 546, 505, 517, 535, 505, 526, 518, 528, 524, 527, 524, 510, 479, 491, 512, 513, 489, 518, 494, 516, 468, 513, 536, 517, 520, 549, 520, 496, 513, 515, 471, 547, 508, 501, 527, 523, 483, 501, 499, 501] payload_rows_per_dst=[5668, 5588, 5552, 5672, 5556, 5446, 5546, 5532] dup_rows_per_dst=[2632, 2546, 2580, 2688, 2604, 2656, 2716, 2554] combine_rows_per_src=[5668, 5588, 5552, 5672, 5556, 5446, 5546, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44538 payload_rows_remote=39018 dup_rows=20998 dup_rows_remote=18342 dedup_ratio=0.320404 router_tokens_per_expert=[464, 499, 497, 515, 505, 516, 522, 496, 522, 488, 500, 521, 502, 505, 524, 555, 522, 533, 494, 489, 545, 517, 499, 499, 529, 544, 517, 517, 500, 529, 502, 513, 519, 538, 532, 517, 483, 531, 490, 478, 497, 527, 529, 545, 505, 552, 516, 554, 518, 510, 493, 517, 506, 498, 501, 493, 509, 503, 503, 522, 493, 498, 514, 497] payload_rows_per_dst=[5514, 5652, 5594, 5640, 5520, 5612, 5524, 5482] dup_rows_per_dst=[2514, 2582, 2602, 2662, 2656, 2838, 2548, 2596] combine_rows_per_src=[5514, 5652, 5594, 5640, 5520, 5612, 5524, 5482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38964 dup_rows=20976 dup_rows_remote=18366 dedup_ratio=0.320068 router_tokens_per_expert=[507, 562, 519, 555, 538, 494, 513, 526, 516, 535, 494, 485, 521, 462, 527, 516, 510, 504, 505, 504, 515, 525, 548, 536, 522, 497, 517, 493, 489, 537, 503, 532, 550, 535, 510, 490, 554, 520, 502, 511, 503, 534, 529, 523, 512, 496, 490, 516, 506, 528, 494, 495, 517, 483, 483, 487, 491, 488, 498, 522, 472, 478, 512, 532] payload_rows_per_dst=[5598, 5574, 5610, 5554, 5656, 5596, 5504, 5468] dup_rows_per_dst=[2830, 2538, 2684, 2626, 2688, 2610, 2482, 2518] combine_rows_per_src=[5598, 5574, 5610, 5554, 5656, 5596, 5504, 5468] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38596 dup_rows=21168 dup_rows_remote=18290 dedup_ratio=0.322998 router_tokens_per_expert=[546, 472, 508, 542, 528, 502, 513, 505, 499, 503, 502, 527, 534, 539, 519, 516, 547, 509, 508, 516, 495, 498, 524, 538, 510, 525, 472, 527, 497, 508, 504, 478, 510, 541, 533, 491, 485, 525, 495, 497, 520, 512, 521, 497, 513, 531, 528, 500, 495, 487, 538, 536, 507, 503, 516, 478, 511, 498, 538, 501, 490, 542, 519, 499] payload_rows_per_dst=[5466, 5618, 5594, 5330, 5496, 5338, 5754, 5772] dup_rows_per_dst=[2552, 2660, 2724, 2286, 2700, 2372, 2996, 2878] combine_rows_per_src=[5466, 5618, 5594, 5330, 5496, 5338, 5754, 5772] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229904 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279079440 ingress_bytes=388950016 link_bytes=388950016 token_bytes=557928976 weight_movement_bytes=110100480 total_nvlink_bytes=668029456 weight_to_token_ratio=0.197338 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764480491e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38888 dup_rows=20976 dup_rows_remote=18288 dedup_ratio=0.320068 router_tokens_per_expert=[517, 515, 496, 509, 529, 550, 527, 507, 479, 545, 489, 519, 509, 525, 500, 501, 510, 530, 506, 510, 481, 508, 521, 500, 546, 505, 517, 535, 505, 526, 518, 528, 524, 527, 524, 510, 479, 491, 512, 513, 489, 518, 494, 516, 468, 513, 536, 517, 520, 549, 520, 496, 513, 515, 471, 547, 508, 501, 527, 523, 483, 501, 499, 501] payload_rows_per_dst=[5668, 5588, 5552, 5672, 5556, 5446, 5546, 5532] dup_rows_per_dst=[2632, 2546, 2580, 2688, 2604, 2656, 2716, 2554] combine_rows_per_src=[5668, 5588, 5552, 5672, 5556, 5446, 5546, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229440 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279910464 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591488 weight_movement_bytes=110100480 total_nvlink_bytes=669691968 weight_to_token_ratio=0.196752 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38914 dup_rows=21108 dup_rows_remote=18476 dedup_ratio=0.322083 router_tokens_per_expert=[524, 493, 513, 498, 533, 483, 499, 501, 531, 518, 500, 521, 509, 499, 492, 470, 514, 518, 526, 509, 512, 518, 511, 499, 498, 495, 504, 529, 530, 494, 531, 511, 488, 559, 496, 527, 542, 524, 530, 511, 508, 535, 532, 505, 515, 507, 519, 490, 520, 533, 503, 522, 515, 468, 484, 528, 516, 539, 516, 506, 504, 517, 525, 501] payload_rows_per_dst=[5478, 5482, 5586, 5600, 5600, 5584, 5514, 5584] dup_rows_per_dst=[2610, 2598, 2628, 2584, 2754, 2638, 2632, 2664] combine_rows_per_src=[5478, 5482, 5586, 5600, 5600, 5584, 5514, 5584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229320 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279523272 ingress_bytes=499494912 link_bytes=499494912 token_bytes=558817224 weight_movement_bytes=220200960 total_nvlink_bytes=779018184 weight_to_token_ratio=0.394048 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687038048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38964 dup_rows=21046 dup_rows_remote=18444 dedup_ratio=0.321136 router_tokens_per_expert=[526, 507, 520, 514, 526, 493, 562, 537, 508, 501, 497, 496, 500, 519, 501, 464, 514, 470, 529, 505, 545, 466, 526, 509, 514, 523, 533, 543, 533, 571, 502, 511, 545, 508, 503, 500, 525, 501, 479, 526, 505, 479, 496, 524, 515, 494, 489, 568, 486, 508, 482, 523, 477, 523, 531, 481, 521, 551, 493, 509, 481, 521, 503, 556] payload_rows_per_dst=[5572, 5458, 5526, 5690, 5596, 5548, 5538, 5562] dup_rows_per_dst=[2798, 2514, 2602, 2770, 2578, 2592, 2484, 2708] combine_rows_per_src=[5572, 5458, 5526, 5690, 5596, 5548, 5538, 5562] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276656128 dispatch_meta_bytes=227544 combine_payload_bytes=276656128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276883672 ingress_bytes=386756608 link_bytes=386756608 token_bytes=553539800 weight_movement_bytes=110100480 total_nvlink_bytes=663640280 weight_to_token_ratio=0.198903 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.726333518e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229320 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279523272 ingress_bytes=499494912 link_bytes=499494912 token_bytes=558817224 weight_movement_bytes=220200960 total_nvlink_bytes=779018184 weight_to_token_ratio=0.394048 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687038048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44408 payload_rows_remote=38810 dup_rows=21128 dup_rows_remote=18506 dedup_ratio=0.322388 router_tokens_per_expert=[526, 511, 547, 530, 485, 475, 496, 540, 486, 520, 551, 526, 524, 541, 497, 502, 533, 506, 495, 510, 516, 534, 523, 507, 543, 506, 497, 504, 499, 494, 528, 499, 495, 509, 512, 473, 474, 520, 531, 522, 540, 502, 543, 509, 476, 513, 560, 539, 515, 492, 512, 512, 475, 543, 504, 499, 549, 497, 499, 503, 498, 501, 509, 491] payload_rows_per_dst=[5598, 5818, 5392, 5522, 5490, 5604, 5512, 5472] dup_rows_per_dst=[2622, 2848, 2484, 2618, 2582, 2760, 2592, 2622] combine_rows_per_src=[5598, 5818, 5392, 5522, 5490, 5604, 5512, 5472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=228704 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278977888 ingress_bytes=498950144 link_bytes=498950144 token_bytes=557727072 weight_movement_bytes=220200960 total_nvlink_bytes=777928032 weight_to_token_ratio=0.394818 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677563636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38964 dup_rows=21046 dup_rows_remote=18444 dedup_ratio=0.321136 router_tokens_per_expert=[526, 507, 520, 514, 526, 493, 562, 537, 508, 501, 497, 496, 500, 519, 501, 464, 514, 470, 529, 505, 545, 466, 526, 509, 514, 523, 533, 543, 533, 571, 502, 511, 545, 508, 503, 500, 525, 501, 479, 526, 505, 479, 496, 524, 515, 494, 489, 568, 486, 508, 482, 523, 477, 523, 531, 481, 521, 551, 493, 509, 481, 521, 503, 556] payload_rows_per_dst=[5572, 5458, 5526, 5690, 5596, 5548, 5538, 5562] dup_rows_per_dst=[2798, 2514, 2602, 2770, 2578, 2592, 2484, 2708] combine_rows_per_src=[5572, 5458, 5526, 5690, 5596, 5548, 5538, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229904 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279079440 ingress_bytes=388950016 link_bytes=388950016 token_bytes=557928976 weight_movement_bytes=110100480 total_nvlink_bytes=668029456 weight_to_token_ratio=0.197338 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764480491e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=229560 combine_payload_bytes=278935552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279165112 ingress_bytes=389036032 link_bytes=389036032 token_bytes=558100664 weight_movement_bytes=110100480 total_nvlink_bytes=668201144 weight_to_token_ratio=0.197277 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765976451e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44408 payload_rows_remote=38838 dup_rows=21128 dup_rows_remote=18468 dedup_ratio=0.322388 router_tokens_per_expert=[487, 513, 517, 540, 504, 527, 500, 541, 528, 513, 563, 517, 493, 476, 500, 525, 520, 512, 518, 508, 467, 518, 515, 523, 503, 505, 537, 531, 522, 526, 472, 523, 484, 523, 558, 519, 450, 550, 541, 516, 503, 503, 494, 515, 518, 516, 508, 471, 542, 510, 519, 540, 526, 515, 509, 500, 504, 508, 530, 509, 420, 512, 517, 494] payload_rows_per_dst=[5622, 5570, 5562, 5542, 5640, 5470, 5594, 5408] dup_rows_per_dst=[2636, 2660, 2600, 2696, 2642, 2586, 2728, 2580] combine_rows_per_src=[5622, 5570, 5562, 5542, 5640, 5470, 5594, 5408] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278190080 dispatch_meta_bytes=229264 combine_payload_bytes=278190080 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278419344 ingress_bytes=718592000 link_bytes=718592000 token_bytes=556609424 weight_movement_bytes=440401920 total_nvlink_bytes=997011344 weight_to_token_ratio=0.791223 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249749676e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44336 payload_rows_remote=38744 dup_rows=21200 dup_rows_remote=18568 dedup_ratio=0.323486 router_tokens_per_expert=[498, 512, 574, 505, 503, 497, 494, 528, 524, 520, 494, 531, 514, 515, 503, 522, 506, 494, 552, 493, 485, 494, 496, 492, 531, 534, 513, 512, 484, 520, 521, 524, 530, 513, 513, 507, 545, 502, 519, 483, 463, 547, 539, 519, 500, 527, 526, 535, 519, 508, 491, 533, 464, 462, 512, 486, 523, 496, 523, 529, 512, 525, 499, 533] payload_rows_per_dst=[5538, 5596, 5462, 5560, 5592, 5574, 5424, 5590] dup_rows_per_dst=[2684, 2650, 2562, 2718, 2632, 2738, 2526, 2690] combine_rows_per_src=[5538, 5596, 5462, 5560, 5592, 5574, 5424, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=228704 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278977888 ingress_bytes=498950144 link_bytes=498950144 token_bytes=557727072 weight_movement_bytes=220200960 total_nvlink_bytes=777928032 weight_to_token_ratio=0.394818 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677563636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229440 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279910464 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591488 weight_movement_bytes=110100480 total_nvlink_bytes=669691968 weight_to_token_ratio=0.196752 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276656128 dispatch_meta_bytes=227544 combine_payload_bytes=276656128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276883672 ingress_bytes=386756608 link_bytes=386756608 token_bytes=553539800 weight_movement_bytes=110100480 total_nvlink_bytes=663640280 weight_to_token_ratio=0.198903 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.726333518e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229632 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279523584 ingress_bytes=499494912 link_bytes=499494912 token_bytes=558817536 weight_movement_bytes=220200960 total_nvlink_bytes=779018496 weight_to_token_ratio=0.394048 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687038048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278190080 dispatch_meta_bytes=229264 combine_payload_bytes=278190080 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278419344 ingress_bytes=718592000 link_bytes=718592000 token_bytes=556609424 weight_movement_bytes=440401920 total_nvlink_bytes=997011344 weight_to_token_ratio=0.791223 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249749676e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38866 dup_rows=21126 dup_rows_remote=18544 dedup_ratio=0.322357 router_tokens_per_expert=[470, 531, 516, 501, 498, 520, 509, 529, 540, 487, 508, 533, 512, 522, 516, 503, 508, 493, 518, 534, 551, 533, 513, 486, 541, 511, 510, 501, 536, 495, 494, 520, 538, 494, 437, 504, 482, 506, 531, 519, 523, 508, 508, 491, 499, 510, 505, 519, 530, 504, 549, 522, 518, 519, 502, 504, 480, 547, 482, 531, 521, 516, 537, 493] payload_rows_per_dst=[5536, 5552, 5590, 5526, 5532, 5544, 5576, 5554] dup_rows_per_dst=[2612, 2690, 2682, 2690, 2490, 2582, 2720, 2660] combine_rows_per_src=[5536, 5552, 5590, 5526, 5532, 5544, 5576, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44620 payload_rows_remote=38880 dup_rows=20916 dup_rows_remote=18044 dedup_ratio=0.319153 router_tokens_per_expert=[510, 527, 501, 506, 539, 490, 543, 483, 545, 500, 512, 517, 497, 525, 543, 523, 481, 480, 541, 527, 500, 496, 515, 536, 526, 507, 510, 488, 549, 535, 484, 509, 520, 533, 528, 511, 471, 484, 547, 461, 486, 519, 519, 510, 503, 508, 521, 537, 521, 536, 529, 512, 522, 535, 494, 502, 499, 501, 482, 530, 483, 524, 519, 476] payload_rows_per_dst=[5480, 5690, 5582, 5410, 5546, 5544, 5628, 5740] dup_rows_per_dst=[2474, 2684, 2570, 2416, 2564, 2654, 2682, 2872] combine_rows_per_src=[5480, 5690, 5582, 5410, 5546, 5544, 5628, 5740] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278390784 dispatch_meta_bytes=229224 combine_payload_bytes=278390784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278620008 ingress_bytes=388491264 link_bytes=388491264 token_bytes=557010792 weight_movement_bytes=110100480 total_nvlink_bytes=667111272 weight_to_token_ratio=0.197663 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756502039e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44408 payload_rows_remote=38838 dup_rows=21128 dup_rows_remote=18468 dedup_ratio=0.322388 router_tokens_per_expert=[487, 513, 517, 540, 504, 527, 500, 541, 528, 513, 563, 517, 493, 476, 500, 525, 520, 512, 518, 508, 467, 518, 515, 523, 503, 505, 537, 531, 522, 526, 472, 523, 484, 523, 558, 519, 450, 550, 541, 516, 503, 503, 494, 515, 518, 516, 508, 471, 542, 510, 519, 540, 526, 515, 509, 500, 504, 508, 530, 509, 420, 512, 517, 494] payload_rows_per_dst=[5622, 5570, 5562, 5542, 5640, 5470, 5594, 5408] dup_rows_per_dst=[2636, 2660, 2600, 2696, 2642, 2586, 2728, 2580] combine_rows_per_src=[5622, 5570, 5562, 5542, 5640, 5470, 5594, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44646 payload_rows_remote=39086 dup_rows=20890 dup_rows_remote=18300 dedup_ratio=0.318756 router_tokens_per_expert=[547, 525, 511, 488, 552, 502, 458, 489, 505, 527, 503, 515, 487, 480, 491, 518, 522, 517, 539, 512, 515, 502, 499, 492, 521, 520, 535, 529, 496, 521, 476, 491, 501, 526, 536, 517, 541, 525, 538, 519, 493, 538, 504, 521, 527, 495, 520, 526, 517, 536, 512, 479, 511, 516, 505, 499, 494, 508, 525, 487, 524, 521, 518, 504] payload_rows_per_dst=[5544, 5484, 5604, 5602, 5654, 5626, 5560, 5572] dup_rows_per_dst=[2600, 2568, 2592, 2576, 2752, 2622, 2590, 2590] combine_rows_per_src=[5544, 5484, 5604, 5602, 5654, 5626, 5560, 5572] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=38944 dup_rows=20918 dup_rows_remote=18190 dedup_ratio=0.319183 router_tokens_per_expert=[552, 534, 489, 501, 517, 515, 488, 517, 529, 551, 484, 496, 483, 506, 532, 509, 514, 506, 514, 538, 505, 503, 495, 486, 490, 524, 557, 548, 524, 519, 501, 538, 519, 545, 505, 495, 476, 467, 527, 497, 538, 478, 545, 503, 514, 500, 521, 518, 540, 529, 524, 547, 517, 490, 522, 496, 474, 545, 499, 492, 505, 506, 462, 507] payload_rows_per_dst=[5620, 5580, 5518, 5674, 5516, 5592, 5690, 5428] dup_rows_per_dst=[2606, 2600, 2604, 2728, 2546, 2642, 2640, 2552] combine_rows_per_src=[5620, 5580, 5518, 5674, 5516, 5592, 5690, 5428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277716992 dispatch_meta_bytes=229248 combine_payload_bytes=277716992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277946240 ingress_bytes=387817472 link_bytes=387817472 token_bytes=555663232 weight_movement_bytes=110100480 total_nvlink_bytes=665763712 weight_to_token_ratio=0.198142 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744783688e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44646 payload_rows_remote=39086 dup_rows=20890 dup_rows_remote=18300 dedup_ratio=0.318756 router_tokens_per_expert=[547, 525, 511, 488, 552, 502, 458, 489, 505, 527, 503, 515, 487, 480, 491, 518, 522, 517, 539, 512, 515, 502, 499, 492, 521, 520, 535, 529, 496, 521, 476, 491, 501, 526, 536, 517, 541, 525, 538, 519, 493, 538, 504, 521, 527, 495, 520, 526, 517, 536, 512, 479, 511, 516, 505, 499, 494, 508, 525, 487, 524, 521, 518, 504] payload_rows_per_dst=[5544, 5484, 5604, 5602, 5654, 5626, 5560, 5572] dup_rows_per_dst=[2600, 2568, 2592, 2576, 2752, 2622, 2590, 2590] combine_rows_per_src=[5544, 5484, 5604, 5602, 5654, 5626, 5560, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278591488 dispatch_meta_bytes=229640 combine_payload_bytes=278591488 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278821128 ingress_bytes=498792448 link_bytes=498792448 token_bytes=557412616 weight_movement_bytes=220200960 total_nvlink_bytes=777613576 weight_to_token_ratio=0.395041 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.674821043e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44620 payload_rows_remote=38880 dup_rows=20916 dup_rows_remote=18044 dedup_ratio=0.319153 router_tokens_per_expert=[510, 527, 501, 506, 539, 490, 543, 483, 545, 500, 512, 517, 497, 525, 543, 523, 481, 480, 541, 527, 500, 496, 515, 536, 526, 507, 510, 488, 549, 535, 484, 509, 520, 533, 528, 511, 471, 484, 547, 461, 486, 519, 519, 510, 503, 508, 521, 537, 521, 536, 529, 512, 522, 535, 494, 502, 499, 501, 482, 530, 483, 524, 519, 476] payload_rows_per_dst=[5480, 5690, 5582, 5410, 5546, 5544, 5628, 5740] dup_rows_per_dst=[2474, 2684, 2570, 2416, 2564, 2654, 2682, 2872] combine_rows_per_src=[5480, 5690, 5582, 5410, 5546, 5544, 5628, 5740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44336 payload_rows_remote=38744 dup_rows=21200 dup_rows_remote=18568 dedup_ratio=0.323486 router_tokens_per_expert=[498, 512, 574, 505, 503, 497, 494, 528, 524, 520, 494, 531, 514, 515, 503, 522, 506, 494, 552, 493, 485, 494, 496, 492, 531, 534, 513, 512, 484, 520, 521, 524, 530, 513, 513, 507, 545, 502, 519, 483, 463, 547, 539, 519, 500, 527, 526, 535, 519, 508, 491, 533, 464, 462, 512, 486, 523, 496, 523, 529, 512, 525, 499, 533] payload_rows_per_dst=[5538, 5596, 5462, 5560, 5592, 5574, 5424, 5590] dup_rows_per_dst=[2684, 2650, 2562, 2718, 2632, 2738, 2526, 2690] combine_rows_per_src=[5538, 5596, 5462, 5560, 5592, 5574, 5424, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278390784 dispatch_meta_bytes=229224 combine_payload_bytes=278390784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278620008 ingress_bytes=388491264 link_bytes=388491264 token_bytes=557010792 weight_movement_bytes=110100480 total_nvlink_bytes=667111272 weight_to_token_ratio=0.197663 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756502039e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=227696 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278919536 ingress_bytes=388792320 link_bytes=388792320 token_bytes=557611376 weight_movement_bytes=110100480 total_nvlink_bytes=667711856 weight_to_token_ratio=0.197450 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761737898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=228536 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279379128 ingress_bytes=499351552 link_bytes=499351552 token_bytes=558529720 weight_movement_bytes=220200960 total_nvlink_bytes=778730680 weight_to_token_ratio=0.394251 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684544781e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44714 payload_rows_remote=39112 dup_rows=20822 dup_rows_remote=18182 dedup_ratio=0.317719 router_tokens_per_expert=[491, 527, 543, 493, 529, 504, 527, 498, 515, 559, 540, 502, 503, 496, 507, 513, 561, 513, 486, 516, 480, 521, 507, 516, 517, 505, 501, 544, 484, 492, 515, 543, 474, 523, 528, 491, 513, 528, 526, 538, 527, 478, 520, 528, 506, 547, 501, 504, 487, 515, 503, 514, 512, 488, 541, 523, 453, 501, 510, 507, 510, 514, 484, 526] payload_rows_per_dst=[5602, 5588, 5672, 5618, 5646, 5584, 5460, 5544] dup_rows_per_dst=[2640, 2682, 2708, 2584, 2686, 2548, 2508, 2466] combine_rows_per_src=[5602, 5588, 5672, 5618, 5646, 5584, 5460, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280168448 dispatch_meta_bytes=229544 combine_payload_bytes=280168448 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280397992 ingress_bytes=610469888 link_bytes=610469888 token_bytes=560566440 weight_movement_bytes=330301440 total_nvlink_bytes=890867880 weight_to_token_ratio=0.589228 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061707540e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38896 dup_rows=21046 dup_rows_remote=18412 dedup_ratio=0.321136 router_tokens_per_expert=[513, 546, 538, 524, 477, 518, 542, 513, 457, 481, 515, 493, 485, 494, 519, 527, 511, 523, 528, 479, 507, 523, 517, 526, 500, 473, 554, 537, 520, 520, 493, 501, 500, 539, 501, 505, 492, 504, 506, 502, 514, 520, 535, 485, 555, 496, 522, 488, 491, 528, 515, 534, 515, 499, 486, 537, 529, 496, 508, 504, 535, 522, 534, 517] payload_rows_per_dst=[5624, 5438, 5594, 5540, 5488, 5582, 5594, 5630] dup_rows_per_dst=[2718, 2504, 2634, 2656, 2610, 2648, 2616, 2660] combine_rows_per_src=[5624, 5438, 5594, 5540, 5488, 5582, 5594, 5630] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=39034 dup_rows=20948 dup_rows_remote=18414 dedup_ratio=0.319641 router_tokens_per_expert=[514, 523, 471, 522, 487, 524, 522, 491, 475, 504, 519, 510, 504, 496, 540, 496, 499, 513, 500, 516, 478, 478, 532, 491, 516, 544, 509, 520, 492, 509, 525, 519, 503, 520, 500, 537, 566, 503, 522, 511, 509, 513, 516, 466, 511, 526, 557, 519, 535, 487, 517, 504, 487, 494, 538, 563, 518, 470, 515, 533, 544, 507, 477, 561] payload_rows_per_dst=[5534, 5554, 5466, 5638, 5626, 5580, 5598, 5592] dup_rows_per_dst=[2574, 2534, 2548, 2630, 2698, 2654, 2652, 2658] combine_rows_per_src=[5534, 5554, 5466, 5638, 5626, 5580, 5598, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280168448 dispatch_meta_bytes=229544 combine_payload_bytes=280168448 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280397992 ingress_bytes=610469888 link_bytes=610469888 token_bytes=560566440 weight_movement_bytes=330301440 total_nvlink_bytes=890867880 weight_to_token_ratio=0.589228 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061707540e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44714 payload_rows_remote=39112 dup_rows=20822 dup_rows_remote=18182 dedup_ratio=0.317719 router_tokens_per_expert=[491, 527, 543, 493, 529, 504, 527, 498, 515, 559, 540, 502, 503, 496, 507, 513, 561, 513, 486, 516, 480, 521, 507, 516, 517, 505, 501, 544, 484, 492, 515, 543, 474, 523, 528, 491, 513, 528, 526, 538, 527, 478, 520, 528, 506, 547, 501, 504, 487, 515, 503, 514, 512, 488, 541, 523, 453, 501, 510, 507, 510, 514, 484, 526] payload_rows_per_dst=[5602, 5588, 5672, 5618, 5646, 5584, 5460, 5544] dup_rows_per_dst=[2640, 2682, 2708, 2584, 2686, 2548, 2508, 2466] combine_rows_per_src=[5602, 5588, 5672, 5618, 5646, 5584, 5460, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=228536 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279379128 ingress_bytes=499351552 link_bytes=499351552 token_bytes=558529720 weight_movement_bytes=220200960 total_nvlink_bytes=778730680 weight_to_token_ratio=0.394251 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684544781e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=227696 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278919536 ingress_bytes=388792320 link_bytes=388792320 token_bytes=557611376 weight_movement_bytes=110100480 total_nvlink_bytes=667711856 weight_to_token_ratio=0.197450 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761737898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277716992 dispatch_meta_bytes=229248 combine_payload_bytes=277716992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277946240 ingress_bytes=387817472 link_bytes=387817472 token_bytes=555663232 weight_movement_bytes=110100480 total_nvlink_bytes=665763712 weight_to_token_ratio=0.198142 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744783688e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=229232 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279035760 ingress_bytes=388907008 link_bytes=388907008 token_bytes=557842288 weight_movement_bytes=110100480 total_nvlink_bytes=667942768 weight_to_token_ratio=0.197368 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763732511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280354816 dispatch_meta_bytes=229176 combine_payload_bytes=280354816 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280583992 ingress_bytes=610656256 link_bytes=610656256 token_bytes=560938808 weight_movement_bytes=330301440 total_nvlink_bytes=891240248 weight_to_token_ratio=0.588837 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062031665e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44400 payload_rows_remote=38820 dup_rows=21136 dup_rows_remote=18446 dedup_ratio=0.322510 router_tokens_per_expert=[524, 498, 509, 554, 487, 506, 492, 544, 511, 514, 503, 494, 494, 488, 481, 498, 492, 528, 540, 513, 478, 526, 486, 493, 546, 480, 521, 561, 467, 513, 537, 532, 519, 497, 502, 522, 520, 525, 517, 533, 554, 499, 517, 532, 540, 501, 551, 493, 516, 479, 532, 497, 515, 526, 500, 500, 526, 536, 520, 518, 518, 498, 465, 490] payload_rows_per_dst=[5624, 5422, 5510, 5618, 5580, 5594, 5502, 5550] dup_rows_per_dst=[2604, 2544, 2602, 2696, 2690, 2780, 2628, 2592] combine_rows_per_src=[5624, 5422, 5510, 5618, 5580, 5594, 5502, 5550] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38912 dup_rows=20966 dup_rows_remote=18286 dedup_ratio=0.319916 router_tokens_per_expert=[485, 547, 516, 556, 517, 523, 536, 532, 489, 536, 499, 491, 558, 494, 480, 527, 525, 532, 488, 478, 547, 497, 513, 520, 503, 506, 514, 544, 522, 513, 484, 535, 488, 511, 530, 496, 491, 528, 476, 492, 503, 541, 520, 522, 540, 510, 505, 528, 495, 512, 526, 505, 505, 513, 464, 511, 480, 507, 541, 467, 500, 526, 511, 517] payload_rows_per_dst=[5630, 5550, 5580, 5622, 5496, 5658, 5492, 5542] dup_rows_per_dst=[2794, 2598, 2620, 2620, 2528, 2680, 2570, 2556] combine_rows_per_src=[5630, 5550, 5580, 5622, 5496, 5658, 5492, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=229232 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279035760 ingress_bytes=388907008 link_bytes=388907008 token_bytes=557842288 weight_movement_bytes=110100480 total_nvlink_bytes=667942768 weight_to_token_ratio=0.197368 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763732511e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44326 payload_rows_remote=38680 dup_rows=21210 dup_rows_remote=18374 dedup_ratio=0.323639 router_tokens_per_expert=[497, 531, 520, 523, 524, 493, 520, 547, 530, 531, 508, 547, 497, 499, 522, 501, 510, 507, 523, 529, 499, 509, 526, 497, 522, 490, 543, 481, 533, 505, 516, 483, 505, 495, 501, 502, 510, 543, 503, 550, 543, 493, 519, 503, 510, 509, 485, 505, 516, 505, 523, 485, 546, 531, 532, 469, 510, 510, 515, 489, 529, 488, 489, 492] payload_rows_per_dst=[5344, 5666, 5572, 5436, 5444, 5580, 5638, 5646] dup_rows_per_dst=[2344, 2788, 2628, 2540, 2580, 2646, 2848, 2836] combine_rows_per_src=[5344, 5666, 5572, 5436, 5444, 5580, 5638, 5646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229792 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280025504 ingress_bytes=389896192 link_bytes=389896192 token_bytes=559821216 weight_movement_bytes=110100480 total_nvlink_bytes=669921696 weight_to_token_ratio=0.196671 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780936048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38812 dup_rows=21134 dup_rows_remote=18500 dedup_ratio=0.322479 router_tokens_per_expert=[506, 518, 508, 500, 530, 485, 511, 522, 488, 507, 515, 527, 499, 532, 506, 518, 513, 516, 500, 490, 507, 562, 491, 532, 520, 533, 525, 507, 520, 500, 488, 519, 507, 524, 535, 529, 493, 541, 500, 542, 519, 529, 510, 490, 529, 540, 498, 495, 522, 507, 456, 546, 522, 481, 500, 514, 526, 494, 479, 503, 511, 521, 504, 506] payload_rows_per_dst=[5544, 5546, 5534, 5590, 5578, 5582, 5504, 5524] dup_rows_per_dst=[2616, 2638, 2688, 2634, 2764, 2638, 2592, 2564] combine_rows_per_src=[5544, 5546, 5534, 5590, 5578, 5582, 5504, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=39034 dup_rows=20948 dup_rows_remote=18414 dedup_ratio=0.319641 router_tokens_per_expert=[514, 523, 471, 522, 487, 524, 522, 491, 475, 504, 519, 510, 504, 496, 540, 496, 499, 513, 500, 516, 478, 478, 532, 491, 516, 544, 509, 520, 492, 509, 525, 519, 503, 520, 500, 537, 566, 503, 522, 511, 509, 513, 516, 466, 511, 526, 557, 519, 535, 487, 517, 504, 487, 494, 538, 563, 518, 470, 515, 533, 544, 507, 477, 561] payload_rows_per_dst=[5534, 5554, 5466, 5638, 5626, 5580, 5598, 5592] dup_rows_per_dst=[2574, 2534, 2548, 2630, 2698, 2654, 2652, 2658] combine_rows_per_src=[5534, 5554, 5466, 5638, 5626, 5580, 5598, 5592] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44766 payload_rows_remote=39188 dup_rows=20770 dup_rows_remote=18188 dedup_ratio=0.316925 router_tokens_per_expert=[501, 535, 533, 523, 554, 550, 519, 499, 501, 506, 520, 545, 521, 522, 494, 518, 538, 507, 522, 489, 493, 546, 504, 523, 519, 530, 507, 483, 506, 514, 500, 502, 506, 473, 512, 531, 512, 534, 528, 517, 503, 473, 479, 456, 486, 517, 511, 499, 533, 536, 485, 478, 518, 535, 497, 498, 507, 512, 534, 525, 486, 500, 549, 514] payload_rows_per_dst=[5694, 5634, 5620, 5586, 5578, 5436, 5578, 5640] dup_rows_per_dst=[2734, 2620, 2624, 2536, 2648, 2412, 2582, 2614] combine_rows_per_src=[5694, 5634, 5620, 5586, 5578, 5436, 5578, 5640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=229064 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278490824 ingress_bytes=498462720 link_bytes=498462720 token_bytes=556752584 weight_movement_bytes=220200960 total_nvlink_bytes=776953544 weight_to_token_ratio=0.395510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669086531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38812 dup_rows=21134 dup_rows_remote=18500 dedup_ratio=0.322479 router_tokens_per_expert=[506, 518, 508, 500, 530, 485, 511, 522, 488, 507, 515, 527, 499, 532, 506, 518, 513, 516, 500, 490, 507, 562, 491, 532, 520, 533, 525, 507, 520, 500, 488, 519, 507, 524, 535, 529, 493, 541, 500, 542, 519, 529, 510, 490, 529, 540, 498, 495, 522, 507, 456, 546, 522, 481, 500, 514, 526, 494, 479, 503, 511, 521, 504, 506] payload_rows_per_dst=[5544, 5546, 5534, 5590, 5578, 5582, 5504, 5524] dup_rows_per_dst=[2616, 2638, 2688, 2634, 2764, 2638, 2592, 2564] combine_rows_per_src=[5544, 5546, 5534, 5590, 5578, 5582, 5504, 5524] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=228792 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279150008 ingress_bytes=389021696 link_bytes=389021696 token_bytes=558071224 weight_movement_bytes=110100480 total_nvlink_bytes=668171704 weight_to_token_ratio=0.197288 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765727124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277258240 dispatch_meta_bytes=228216 combine_payload_bytes=277258240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277486456 ingress_bytes=387358720 link_bytes=387358720 token_bytes=554744696 weight_movement_bytes=110100480 total_nvlink_bytes=664845176 weight_to_token_ratio=0.198471 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.736805236e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44326 payload_rows_remote=38680 dup_rows=21210 dup_rows_remote=18374 dedup_ratio=0.323639 router_tokens_per_expert=[497, 531, 520, 523, 524, 493, 520, 547, 530, 531, 508, 547, 497, 499, 522, 501, 510, 507, 523, 529, 499, 509, 526, 497, 522, 490, 543, 481, 533, 505, 516, 483, 505, 495, 501, 502, 510, 543, 503, 550, 543, 493, 519, 503, 510, 509, 485, 505, 516, 505, 523, 485, 546, 531, 532, 469, 510, 510, 515, 489, 529, 488, 489, 492] payload_rows_per_dst=[5344, 5666, 5572, 5436, 5444, 5580, 5638, 5646] dup_rows_per_dst=[2344, 2788, 2628, 2540, 2580, 2646, 2848, 2836] combine_rows_per_src=[5344, 5666, 5572, 5436, 5444, 5580, 5638, 5646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44400 payload_rows_remote=38820 dup_rows=21136 dup_rows_remote=18446 dedup_ratio=0.322510 router_tokens_per_expert=[524, 498, 509, 554, 487, 506, 492, 544, 511, 514, 503, 494, 494, 488, 481, 498, 492, 528, 540, 513, 478, 526, 486, 493, 546, 480, 521, 561, 467, 513, 537, 532, 519, 497, 502, 522, 520, 525, 517, 533, 554, 499, 517, 532, 540, 501, 551, 493, 516, 479, 532, 497, 515, 526, 500, 500, 526, 536, 520, 518, 518, 498, 465, 490] payload_rows_per_dst=[5624, 5422, 5510, 5618, 5580, 5594, 5502, 5550] dup_rows_per_dst=[2604, 2544, 2602, 2696, 2690, 2780, 2628, 2592] combine_rows_per_src=[5624, 5422, 5510, 5618, 5580, 5594, 5502, 5550] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229792 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280025504 ingress_bytes=389896192 link_bytes=389896192 token_bytes=559821216 weight_movement_bytes=110100480 total_nvlink_bytes=669921696 weight_to_token_ratio=0.196671 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780936048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39010 dup_rows=20968 dup_rows_remote=18398 dedup_ratio=0.319946 router_tokens_per_expert=[520, 515, 521, 535, 541, 512, 490, 506, 499, 544, 498, 484, 510, 548, 559, 486, 514, 474, 514, 525, 508, 511, 514, 504, 496, 538, 530, 560, 506, 536, 499, 525, 505, 492, 519, 503, 504, 521, 501, 501, 512, 507, 548, 449, 532, 498, 502, 537, 501, 497, 523, 485, 494, 527, 519, 529, 467, 510, 517, 502, 494, 519, 517, 514] payload_rows_per_dst=[5636, 5564, 5558, 5650, 5576, 5574, 5560, 5450] dup_rows_per_dst=[2644, 2692, 2570, 2730, 2516, 2596, 2590, 2630] combine_rows_per_src=[5636, 5564, 5558, 5650, 5576, 5574, 5560, 5450] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44318 payload_rows_remote=38772 dup_rows=21218 dup_rows_remote=18544 dedup_ratio=0.323761 router_tokens_per_expert=[541, 496, 456, 502, 519, 544, 503, 548, 516, 516, 491, 545, 514, 496, 512, 503, 545, 527, 491, 505, 487, 514, 517, 461, 526, 514, 523, 567, 529, 466, 534, 508, 499, 525, 488, 493, 511, 493, 573, 535, 498, 515, 507, 516, 486, 514, 494, 533, 496, 512, 518, 524, 537, 506, 493, 515, 497, 495, 527, 520, 506, 518, 481, 527] payload_rows_per_dst=[5546, 5526, 5636, 5462, 5554, 5516, 5598, 5480] dup_rows_per_dst=[2674, 2660, 2780, 2548, 2680, 2610, 2604, 2662] combine_rows_per_src=[5546, 5526, 5636, 5462, 5554, 5516, 5598, 5480] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278204416 dispatch_meta_bytes=229248 combine_payload_bytes=278204416 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278433664 ingress_bytes=718606336 link_bytes=718606336 token_bytes=556638080 weight_movement_bytes=440401920 total_nvlink_bytes=997040000 weight_to_token_ratio=0.791182 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249774609e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280899584 dispatch_meta_bytes=229504 combine_payload_bytes=280899584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281129088 ingress_bytes=391000064 link_bytes=391000064 token_bytes=562028672 weight_movement_bytes=110100480 total_nvlink_bytes=672129152 weight_to_token_ratio=0.195898 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.800134198e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278204416 dispatch_meta_bytes=229248 combine_payload_bytes=278204416 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278433664 ingress_bytes=718606336 link_bytes=718606336 token_bytes=556638080 weight_movement_bytes=440401920 total_nvlink_bytes=997040000 weight_to_token_ratio=0.791182 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249774609e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277258240 dispatch_meta_bytes=228216 combine_payload_bytes=277258240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277486456 ingress_bytes=387358720 link_bytes=387358720 token_bytes=554744696 weight_movement_bytes=110100480 total_nvlink_bytes=664845176 weight_to_token_ratio=0.198471 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.736805236e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=229064 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278490824 ingress_bytes=498462720 link_bytes=498462720 token_bytes=556752584 weight_movement_bytes=220200960 total_nvlink_bytes=776953544 weight_to_token_ratio=0.395510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669086531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44620 payload_rows_remote=39018 dup_rows=20916 dup_rows_remote=18262 dedup_ratio=0.319153 router_tokens_per_expert=[480, 507, 498, 519, 502, 525, 510, 489, 495, 505, 515, 501, 513, 545, 531, 523, 489, 518, 535, 549, 571, 508, 523, 507, 513, 516, 502, 538, 526, 500, 521, 500, 496, 505, 534, 498, 499, 467, 531, 529, 504, 497, 521, 541, 478, 518, 513, 504, 511, 532, 497, 528, 505, 452, 526, 495, 519, 499, 503, 530, 475, 506, 550, 531] payload_rows_per_dst=[5518, 5602, 5630, 5574, 5590, 5570, 5546, 5590] dup_rows_per_dst=[2542, 2654, 2770, 2658, 2528, 2582, 2546, 2636] combine_rows_per_src=[5518, 5602, 5630, 5574, 5590, 5570, 5546, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39010 dup_rows=20968 dup_rows_remote=18398 dedup_ratio=0.319946 router_tokens_per_expert=[520, 515, 521, 535, 541, 512, 490, 506, 499, 544, 498, 484, 510, 548, 559, 486, 514, 474, 514, 525, 508, 511, 514, 504, 496, 538, 530, 560, 506, 536, 499, 525, 505, 492, 519, 503, 504, 521, 501, 501, 512, 507, 548, 449, 532, 498, 502, 537, 501, 497, 523, 485, 494, 527, 519, 529, 467, 510, 517, 502, 494, 519, 517, 514] payload_rows_per_dst=[5636, 5564, 5558, 5650, 5576, 5574, 5560, 5450] dup_rows_per_dst=[2644, 2692, 2570, 2730, 2516, 2596, 2590, 2630] combine_rows_per_src=[5636, 5564, 5558, 5650, 5576, 5574, 5560, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=229632 combine_payload_bytes=279623680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279853312 ingress_bytes=389724160 link_bytes=389724160 token_bytes=559476992 weight_movement_bytes=110100480 total_nvlink_bytes=669577472 weight_to_token_ratio=0.196792 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777944128e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=229264 combine_payload_bytes=277917696 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278146960 ingress_bytes=608219136 link_bytes=608219136 token_bytes=556064656 weight_movement_bytes=330301440 total_nvlink_bytes=886366096 weight_to_token_ratio=0.593998 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057793112e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38722 dup_rows=21158 dup_rows_remote=18510 dedup_ratio=0.322845 router_tokens_per_expert=[514, 512, 516, 516, 534, 491, 511, 517, 535, 487, 514, 470, 518, 515, 513, 480, 500, 470, 508, 516, 488, 517, 518, 567, 566, 573, 518, 522, 514, 496, 476, 509, 519, 507, 537, 508, 532, 511, 536, 502, 482, 523, 490, 511, 518, 511, 515, 507, 527, 488, 526, 526, 492, 474, 507, 554, 483, 538, 528, 523, 525, 493, 499, 475] payload_rows_per_dst=[5518, 5532, 5530, 5600, 5656, 5536, 5484, 5522] dup_rows_per_dst=[2704, 2532, 2638, 2748, 2648, 2578, 2704, 2606] combine_rows_per_src=[5518, 5532, 5530, 5600, 5656, 5536, 5484, 5522] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44612 payload_rows_remote=39144 dup_rows=20924 dup_rows_remote=18430 dedup_ratio=0.319275 router_tokens_per_expert=[516, 504, 480, 496, 499, 539, 528, 515, 548, 509, 522, 518, 513, 528, 513, 541, 530, 526, 513, 525, 497, 543, 511, 495, 493, 491, 517, 493, 509, 559, 517, 506, 511, 474, 500, 495, 508, 516, 535, 530, 484, 509, 532, 492, 500, 463, 477, 524, 525, 493, 475, 547, 515, 508, 505, 475, 548, 498, 525, 520, 508, 530, 518, 534] payload_rows_per_dst=[5568, 5666, 5582, 5592, 5580, 5468, 5466, 5690] dup_rows_per_dst=[2586, 2718, 2698, 2578, 2558, 2494, 2620, 2672] combine_rows_per_src=[5568, 5666, 5582, 5592, 5580, 5468, 5466, 5690] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229120 combine_payload_bytes=279681024 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279910144 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591168 weight_movement_bytes=110100480 total_nvlink_bytes=669691648 weight_to_token_ratio=0.196752 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=39010 dup_rows=20940 dup_rows_remote=18312 dedup_ratio=0.319519 router_tokens_per_expert=[507, 533, 519, 512, 521, 541, 519, 523, 484, 495, 528, 535, 508, 488, 500, 533, 538, 523, 525, 552, 465, 514, 478, 489, 503, 511, 492, 485, 499, 474, 538, 517, 474, 533, 557, 502, 521, 521, 560, 539, 468, 480, 531, 508, 480, 506, 511, 564, 553, 451, 538, 500, 472, 558, 503, 484, 490, 533, 533, 496, 490, 537, 488, 538] payload_rows_per_dst=[5672, 5532, 5600, 5534, 5414, 5528, 5730, 5586] dup_rows_per_dst=[2678, 2606, 2568, 2504, 2558, 2568, 2830, 2628] combine_rows_per_src=[5672, 5532, 5600, 5534, 5414, 5528, 5730, 5586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=229632 combine_payload_bytes=279623680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279853312 ingress_bytes=389724160 link_bytes=389724160 token_bytes=559476992 weight_movement_bytes=110100480 total_nvlink_bytes=669577472 weight_to_token_ratio=0.196792 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777944128e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44620 payload_rows_remote=39018 dup_rows=20916 dup_rows_remote=18262 dedup_ratio=0.319153 router_tokens_per_expert=[480, 507, 498, 519, 502, 525, 510, 489, 495, 505, 515, 501, 513, 545, 531, 523, 489, 518, 535, 549, 571, 508, 523, 507, 513, 516, 502, 538, 526, 500, 521, 500, 496, 505, 534, 498, 499, 467, 531, 529, 504, 497, 521, 541, 478, 518, 513, 504, 511, 532, 497, 528, 505, 452, 526, 495, 519, 499, 503, 530, 475, 506, 550, 531] payload_rows_per_dst=[5518, 5602, 5630, 5574, 5590, 5570, 5546, 5590] dup_rows_per_dst=[2542, 2654, 2770, 2658, 2528, 2582, 2546, 2636] combine_rows_per_src=[5518, 5602, 5630, 5574, 5590, 5570, 5546, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280584192 dispatch_meta_bytes=230296 combine_payload_bytes=280584192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280814488 ingress_bytes=390684672 link_bytes=390684672 token_bytes=561398680 weight_movement_bytes=110100480 total_nvlink_bytes=671499160 weight_to_token_ratio=0.196118 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794649012e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38794 dup_rows=21092 dup_rows_remote=18408 dedup_ratio=0.321838 router_tokens_per_expert=[525, 501, 514, 531, 474, 500, 476, 517, 515, 546, 517, 510, 474, 493, 491, 486, 506, 460, 512, 514, 522, 523, 482, 473, 553, 516, 516, 505, 516, 557, 490, 514, 507, 535, 551, 511, 534, 554, 494, 491, 516, 511, 546, 498, 525, 519, 540, 551, 533, 515, 503, 492, 467, 515, 491, 523, 494, 480, 537, 556, 495, 516, 516, 523] payload_rows_per_dst=[5502, 5460, 5520, 5650, 5628, 5606, 5518, 5560] dup_rows_per_dst=[2574, 2604, 2464, 2684, 2726, 2806, 2560, 2674] combine_rows_per_src=[5502, 5460, 5520, 5650, 5628, 5606, 5518, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277559296 dispatch_meta_bytes=228928 combine_payload_bytes=277559296 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277788224 ingress_bytes=497760256 link_bytes=497760256 token_bytes=555347520 weight_movement_bytes=220200960 total_nvlink_bytes=775548480 weight_to_token_ratio=0.396510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.656869527e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=39010 dup_rows=20940 dup_rows_remote=18312 dedup_ratio=0.319519 router_tokens_per_expert=[507, 533, 519, 512, 521, 541, 519, 523, 484, 495, 528, 535, 508, 488, 500, 533, 538, 523, 525, 552, 465, 514, 478, 489, 503, 511, 492, 485, 499, 474, 538, 517, 474, 533, 557, 502, 521, 521, 560, 539, 468, 480, 531, 508, 480, 506, 511, 564, 553, 451, 538, 500, 472, 558, 503, 484, 490, 533, 533, 496, 490, 537, 488, 538] payload_rows_per_dst=[5672, 5532, 5600, 5534, 5414, 5528, 5730, 5586] dup_rows_per_dst=[2678, 2606, 2568, 2504, 2558, 2568, 2830, 2628] combine_rows_per_src=[5672, 5532, 5600, 5534, 5414, 5528, 5730, 5586] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38960 dup_rows=20988 dup_rows_remote=18372 dedup_ratio=0.320251 router_tokens_per_expert=[504, 515, 533, 457, 489, 495, 525, 505, 541, 515, 513, 478, 521, 515, 492, 500, 532, 475, 500, 508, 509, 492, 512, 517, 524, 547, 492, 502, 511, 521, 520, 534, 513, 492, 509, 532, 525, 495, 506, 509, 527, 512, 524, 542, 537, 534, 511, 554, 497, 468, 532, 556, 500, 487, 506, 556, 511, 527, 501, 539, 470, 519, 493, 490] payload_rows_per_dst=[5468, 5512, 5548, 5594, 5596, 5726, 5588, 5516] dup_rows_per_dst=[2578, 2638, 2542, 2708, 2566, 2756, 2616, 2584] combine_rows_per_src=[5468, 5512, 5548, 5594, 5596, 5726, 5588, 5516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280584192 dispatch_meta_bytes=230296 combine_payload_bytes=280584192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280814488 ingress_bytes=390684672 link_bytes=390684672 token_bytes=561398680 weight_movement_bytes=110100480 total_nvlink_bytes=671499160 weight_to_token_ratio=0.196118 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794649012e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38722 dup_rows=21158 dup_rows_remote=18510 dedup_ratio=0.322845 router_tokens_per_expert=[514, 512, 516, 516, 534, 491, 511, 517, 535, 487, 514, 470, 518, 515, 513, 480, 500, 470, 508, 516, 488, 517, 518, 567, 566, 573, 518, 522, 514, 496, 476, 509, 519, 507, 537, 508, 532, 511, 536, 502, 482, 523, 490, 511, 518, 511, 515, 507, 527, 488, 526, 526, 492, 474, 507, 554, 483, 538, 528, 523, 525, 493, 499, 475] payload_rows_per_dst=[5518, 5532, 5530, 5600, 5656, 5536, 5484, 5522] dup_rows_per_dst=[2704, 2532, 2638, 2748, 2648, 2578, 2704, 2606] combine_rows_per_src=[5518, 5532, 5530, 5600, 5656, 5536, 5484, 5522] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=229288 combine_payload_bytes=279623680 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279852968 ingress_bytes=720025600 link_bytes=720025600 token_bytes=559476648 weight_movement_bytes=440401920 total_nvlink_bytes=999878568 weight_to_token_ratio=0.787168 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252242942e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=38804 dup_rows=20990 dup_rows_remote=18208 dedup_ratio=0.320282 router_tokens_per_expert=[511, 513, 536, 546, 535, 525, 528, 541, 521, 497, 547, 503, 514, 498, 479, 540, 540, 460, 507, 508, 522, 555, 475, 517, 486, 503, 501, 482, 508, 521, 513, 529, 526, 497, 510, 506, 506, 501, 527, 512, 486, 489, 522, 508, 540, 535, 477, 503, 460, 504, 515, 484, 538, 526, 493, 520, 565, 537, 489, 517, 498, 520, 489, 507] payload_rows_per_dst=[5742, 5564, 5646, 5450, 5630, 5516, 5514, 5484] dup_rows_per_dst=[2782, 2634, 2720, 2400, 2626, 2614, 2566, 2648] combine_rows_per_src=[5742, 5564, 5646, 5450, 5630, 5516, 5514, 5484] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229120 combine_payload_bytes=279681024 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279910144 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591168 weight_movement_bytes=110100480 total_nvlink_bytes=669691648 weight_to_token_ratio=0.196752 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44456 payload_rows_remote=39020 dup_rows=21080 dup_rows_remote=18436 dedup_ratio=0.321655 router_tokens_per_expert=[490, 497, 521, 509, 514, 497, 525, 499, 487, 526, 485, 547, 518, 536, 522, 520, 503, 511, 502, 481, 538, 492, 517, 496, 524, 530, 528, 501, 532, 518, 488, 510, 551, 508, 520, 526, 479, 498, 530, 504, 494, 501, 551, 518, 510, 471, 506, 543, 509, 490, 536, 505, 505, 554, 516, 509, 493, 535, 493, 532, 521, 519, 473, 504] payload_rows_per_dst=[5518, 5642, 5436, 5650, 5528, 5556, 5486, 5640] dup_rows_per_dst=[2586, 2640, 2644, 2612, 2704, 2632, 2762, 2500] combine_rows_per_src=[5518, 5642, 5436, 5650, 5528, 5556, 5486, 5640] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278075392 dispatch_meta_bytes=228808 combine_payload_bytes=278075392 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278304200 ingress_bytes=608376832 link_bytes=608376832 token_bytes=556379592 weight_movement_bytes=330301440 total_nvlink_bytes=886681032 weight_to_token_ratio=0.593662 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058067372e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=229288 combine_payload_bytes=279623680 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279852968 ingress_bytes=720025600 link_bytes=720025600 token_bytes=559476648 weight_movement_bytes=440401920 total_nvlink_bytes=999878568 weight_to_token_ratio=0.787168 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252242942e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=229328 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279494608 ingress_bytes=389365760 link_bytes=389365760 token_bytes=558759888 weight_movement_bytes=110100480 total_nvlink_bytes=668860368 weight_to_token_ratio=0.197044 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771710963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39036 dup_rows=20968 dup_rows_remote=18432 dedup_ratio=0.319946 router_tokens_per_expert=[512, 488, 507, 501, 532, 485, 520, 517, 530, 481, 526, 495, 520, 512, 479, 491, 479, 535, 536, 516, 541, 494, 541, 542, 540, 490, 563, 527, 552, 494, 504, 496, 493, 513, 531, 487, 485, 511, 528, 513, 540, 533, 509, 526, 493, 515, 503, 528, 511, 501, 545, 491, 512, 532, 477, 504, 513, 481, 512, 500, 517, 538, 508, 472] payload_rows_per_dst=[5538, 5532, 5696, 5606, 5520, 5630, 5568, 5478] dup_rows_per_dst=[2586, 2536, 2672, 2726, 2602, 2664, 2578, 2604] combine_rows_per_src=[5538, 5532, 5696, 5606, 5520, 5630, 5568, 5478] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=228048 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278375120 ingress_bytes=388247552 link_bytes=388247552 token_bytes=556522192 weight_movement_bytes=110100480 total_nvlink_bytes=666622672 weight_to_token_ratio=0.197837 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752263487e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44456 payload_rows_remote=39020 dup_rows=21080 dup_rows_remote=18436 dedup_ratio=0.321655 router_tokens_per_expert=[490, 497, 521, 509, 514, 497, 525, 499, 487, 526, 485, 547, 518, 536, 522, 520, 503, 511, 502, 481, 538, 492, 517, 496, 524, 530, 528, 501, 532, 518, 488, 510, 551, 508, 520, 526, 479, 498, 530, 504, 494, 501, 551, 518, 510, 471, 506, 543, 509, 490, 536, 505, 505, 554, 516, 509, 493, 535, 493, 532, 521, 519, 473, 504] payload_rows_per_dst=[5518, 5642, 5436, 5650, 5528, 5556, 5486, 5640] dup_rows_per_dst=[2586, 2640, 2644, 2612, 2704, 2632, 2762, 2500] combine_rows_per_src=[5518, 5642, 5436, 5650, 5528, 5556, 5486, 5640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44672 payload_rows_remote=39112 dup_rows=20864 dup_rows_remote=18280 dedup_ratio=0.318359 router_tokens_per_expert=[525, 519, 515, 494, 507, 574, 477, 538, 471, 531, 486, 495, 463, 520, 506, 549, 473, 505, 528, 499, 530, 473, 485, 504, 528, 479, 516, 491, 533, 529, 502, 570, 493, 524, 525, 510, 489, 488, 530, 513, 501, 509, 501, 529, 497, 513, 523, 540, 534, 520, 509, 509, 499, 493, 539, 534, 507, 518, 541, 514, 521, 545, 473, 512] payload_rows_per_dst=[5654, 5498, 5538, 5606, 5560, 5646, 5586, 5584] dup_rows_per_dst=[2644, 2544, 2456, 2690, 2584, 2580, 2688, 2678] combine_rows_per_src=[5654, 5498, 5538, 5606, 5560, 5646, 5586, 5584] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277559296 dispatch_meta_bytes=228928 combine_payload_bytes=277559296 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277788224 ingress_bytes=497760256 link_bytes=497760256 token_bytes=555347520 weight_movement_bytes=220200960 total_nvlink_bytes=775548480 weight_to_token_ratio=0.396510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.656869527e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=228048 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278375120 ingress_bytes=388247552 link_bytes=388247552 token_bytes=556522192 weight_movement_bytes=110100480 total_nvlink_bytes=666622672 weight_to_token_ratio=0.197837 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752263487e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44332 payload_rows_remote=38796 dup_rows=21204 dup_rows_remote=18602 dedup_ratio=0.323547 router_tokens_per_expert=[564, 496, 529, 502, 503, 521, 525, 494, 503, 507, 502, 541, 505, 522, 492, 527, 532, 523, 528, 487, 506, 551, 533, 515, 511, 512, 488, 501, 488, 501, 538, 500, 507, 516, 515, 493, 475, 496, 491, 509, 491, 504, 523, 536, 515, 503, 494, 503, 510, 535, 517, 550, 505, 506, 528, 506, 506, 503, 504, 510, 506, 486, 541, 537] payload_rows_per_dst=[5582, 5556, 5630, 5508, 5382, 5536, 5544, 5594] dup_rows_per_dst=[2686, 2642, 2720, 2570, 2622, 2602, 2770, 2592] combine_rows_per_src=[5582, 5556, 5630, 5508, 5382, 5536, 5544, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279810048 dispatch_meta_bytes=229872 combine_payload_bytes=279810048 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280039920 ingress_bytes=500011008 link_bytes=500011008 token_bytes=559849968 weight_movement_bytes=220200960 total_nvlink_bytes=780050928 weight_to_token_ratio=0.393321 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.696013806e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38784 dup_rows=21166 dup_rows_remote=18448 dedup_ratio=0.322968 router_tokens_per_expert=[479, 549, 534, 565, 482, 518, 518, 532, 488, 523, 496, 483, 506, 521, 509, 552, 492, 529, 495, 494, 517, 500, 552, 528, 536, 522, 493, 518, 510, 503, 511, 517, 525, 555, 500, 484, 488, 499, 492, 524, 491, 531, 531, 529, 508, 496, 492, 545, 507, 469, 499, 526, 502, 532, 521, 503, 526, 499, 498, 511, 511, 533, 473, 496] payload_rows_per_dst=[5628, 5576, 5262, 5524, 5716, 5620, 5458, 5586] dup_rows_per_dst=[2726, 2464, 2290, 2676, 3080, 2742, 2470, 2718] combine_rows_per_src=[5628, 5576, 5262, 5524, 5716, 5620, 5458, 5586] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279695360 dispatch_meta_bytes=229824 combine_payload_bytes=279695360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279925184 ingress_bytes=389795840 link_bytes=389795840 token_bytes=559620544 weight_movement_bytes=110100480 total_nvlink_bytes=669721024 weight_to_token_ratio=0.196741 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779190762e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38846 dup_rows=21076 dup_rows_remote=18490 dedup_ratio=0.321594 router_tokens_per_expert=[523, 528, 497, 529, 513, 535, 518, 474, 540, 528, 534, 495, 518, 478, 538, 503, 501, 504, 511, 517, 507, 525, 528, 572, 509, 514, 538, 478, 531, 523, 511, 496, 458, 501, 494, 480, 512, 501, 477, 496, 539, 514, 510, 525, 533, 500, 477, 502, 517, 539, 519, 500, 500, 532, 527, 512, 514, 531, 486, 493, 505, 522, 512, 524] payload_rows_per_dst=[5572, 5584, 5582, 5614, 5326, 5594, 5596, 5592] dup_rows_per_dst=[2662, 2684, 2748, 2586, 2512, 2606, 2696, 2582] combine_rows_per_src=[5572, 5584, 5582, 5614, 5326, 5594, 5596, 5592] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39036 dup_rows=20968 dup_rows_remote=18432 dedup_ratio=0.319946 router_tokens_per_expert=[512, 488, 507, 501, 532, 485, 520, 517, 530, 481, 526, 495, 520, 512, 479, 491, 479, 535, 536, 516, 541, 494, 541, 542, 540, 490, 563, 527, 552, 494, 504, 496, 493, 513, 531, 487, 485, 511, 528, 513, 540, 533, 509, 526, 493, 515, 503, 528, 511, 501, 545, 491, 512, 532, 477, 504, 513, 481, 512, 500, 517, 538, 508, 472] payload_rows_per_dst=[5538, 5532, 5696, 5606, 5520, 5630, 5568, 5478] dup_rows_per_dst=[2586, 2536, 2672, 2726, 2602, 2664, 2578, 2604] combine_rows_per_src=[5538, 5532, 5696, 5606, 5520, 5630, 5568, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38846 dup_rows=21076 dup_rows_remote=18490 dedup_ratio=0.321594 router_tokens_per_expert=[523, 528, 497, 529, 513, 535, 518, 474, 540, 528, 534, 495, 518, 478, 538, 503, 501, 504, 511, 517, 507, 525, 528, 572, 509, 514, 538, 478, 531, 523, 511, 496, 458, 501, 494, 480, 512, 501, 477, 496, 539, 514, 510, 525, 533, 500, 477, 502, 517, 539, 519, 500, 500, 532, 527, 512, 514, 531, 486, 493, 505, 522, 512, 524] payload_rows_per_dst=[5572, 5584, 5582, 5614, 5326, 5594, 5596, 5592] dup_rows_per_dst=[2662, 2684, 2748, 2586, 2512, 2606, 2696, 2582] combine_rows_per_src=[5572, 5584, 5582, 5614, 5326, 5594, 5596, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278089728 dispatch_meta_bytes=229592 combine_payload_bytes=278089728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278319320 ingress_bytes=388190208 link_bytes=388190208 token_bytes=556409048 weight_movement_bytes=110100480 total_nvlink_bytes=666509528 weight_to_token_ratio=0.197877 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751266180e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38784 dup_rows=21166 dup_rows_remote=18448 dedup_ratio=0.322968 router_tokens_per_expert=[479, 549, 534, 565, 482, 518, 518, 532, 488, 523, 496, 483, 506, 521, 509, 552, 492, 529, 495, 494, 517, 500, 552, 528, 536, 522, 493, 518, 510, 503, 511, 517, 525, 555, 500, 484, 488, 499, 492, 524, 491, 531, 531, 529, 508, 496, 492, 545, 507, 469, 499, 526, 502, 532, 521, 503, 526, 499, 498, 511, 511, 533, 473, 496] payload_rows_per_dst=[5628, 5576, 5262, 5524, 5716, 5620, 5458, 5586] dup_rows_per_dst=[2726, 2464, 2290, 2676, 3080, 2742, 2470, 2718] combine_rows_per_src=[5628, 5576, 5262, 5524, 5716, 5620, 5458, 5586] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38932 dup_rows=20966 dup_rows_remote=18372 dedup_ratio=0.319916 router_tokens_per_expert=[484, 512, 494, 499, 480, 509, 462, 496, 545, 572, 535, 500, 507, 532, 535, 507, 516, 538, 523, 549, 526, 530, 534, 510, 530, 506, 529, 509, 539, 477, 481, 514, 547, 536, 512, 510, 487, 510, 501, 503, 523, 518, 502, 491, 488, 473, 492, 526, 508, 512, 557, 510, 525, 505, 492, 507, 499, 509, 504, 486, 482, 511, 547, 515] payload_rows_per_dst=[5420, 5650, 5674, 5602, 5516, 5500, 5638, 5570] dup_rows_per_dst=[2452, 2816, 2778, 2568, 2696, 2526, 2594, 2536] combine_rows_per_src=[5420, 5650, 5674, 5602, 5516, 5500, 5638, 5570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280354816 dispatch_meta_bytes=229568 combine_payload_bytes=280354816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280584384 ingress_bytes=390455296 link_bytes=390455296 token_bytes=560939200 weight_movement_bytes=110100480 total_nvlink_bytes=671039680 weight_to_token_ratio=0.196279 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790659786e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44714 payload_rows_remote=39066 dup_rows=20822 dup_rows_remote=18178 dedup_ratio=0.317719 router_tokens_per_expert=[550, 503, 514, 482, 511, 497, 542, 539, 516, 534, 513, 519, 536, 517, 521, 466, 539, 529, 493, 515, 495, 530, 546, 485, 546, 495, 494, 519, 506, 509, 481, 480, 496, 478, 581, 515, 493, 488, 524, 513, 476, 542, 491, 531, 536, 508, 516, 457, 486, 504, 474, 517, 532, 545, 511, 521, 514, 501, 512, 502, 539, 508, 515, 520] payload_rows_per_dst=[5648, 5632, 5608, 5500, 5560, 5562, 5602, 5602] dup_rows_per_dst=[2644, 2632, 2740, 2524, 2616, 2552, 2494, 2620] combine_rows_per_src=[5648, 5632, 5608, 5500, 5560, 5562, 5602, 5602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44708 payload_rows_remote=39130 dup_rows=20828 dup_rows_remote=18222 dedup_ratio=0.317810 router_tokens_per_expert=[477, 480, 504, 535, 539, 514, 518, 503, 513, 471, 535, 529, 526, 493, 516, 509, 467, 512, 486, 516, 501, 511, 482, 500, 509, 511, 542, 529, 491, 496, 491, 495, 509, 524, 544, 557, 503, 497, 509, 543, 491, 502, 525, 537, 536, 538, 536, 536, 522, 508, 524, 525, 503, 498, 478, 524, 483, 529, 515, 522, 496, 519, 529, 505] payload_rows_per_dst=[5576, 5578, 5500, 5550, 5706, 5592, 5562, 5644] dup_rows_per_dst=[2564, 2606, 2450, 2578, 2666, 2810, 2602, 2552] combine_rows_per_src=[5576, 5578, 5500, 5550, 5706, 5592, 5562, 5644] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39062 dup_rows=20968 dup_rows_remote=18332 dedup_ratio=0.319946 router_tokens_per_expert=[472, 523, 527, 487, 531, 456, 556, 495, 522, 533, 484, 503, 535, 521, 503, 494, 502, 525, 507, 534, 514, 516, 464, 509, 502, 537, 515, 491, 518, 502, 520, 533, 512, 500, 525, 489, 509, 535, 538, 514, 495, 524, 527, 538, 516, 500, 510, 478, 484, 540, 537, 524, 545, 504, 500, 493, 491, 548, 520, 535, 514, 493, 517, 482] payload_rows_per_dst=[5556, 5598, 5506, 5590, 5564, 5566, 5648, 5540] dup_rows_per_dst=[2538, 2592, 2636, 2646, 2680, 2610, 2606, 2660] combine_rows_per_src=[5556, 5598, 5506, 5590, 5564, 5566, 5648, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278089728 dispatch_meta_bytes=229592 combine_payload_bytes=278089728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278319320 ingress_bytes=388190208 link_bytes=388190208 token_bytes=556409048 weight_movement_bytes=110100480 total_nvlink_bytes=666509528 weight_to_token_ratio=0.197877 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751266180e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44714 payload_rows_remote=39066 dup_rows=20822 dup_rows_remote=18178 dedup_ratio=0.317719 router_tokens_per_expert=[550, 503, 514, 482, 511, 497, 542, 539, 516, 534, 513, 519, 536, 517, 521, 466, 539, 529, 493, 515, 495, 530, 546, 485, 546, 495, 494, 519, 506, 509, 481, 480, 496, 478, 581, 515, 493, 488, 524, 513, 476, 542, 491, 531, 536, 508, 516, 457, 486, 504, 474, 517, 532, 545, 511, 521, 514, 501, 512, 502, 539, 508, 515, 520] payload_rows_per_dst=[5648, 5632, 5608, 5500, 5560, 5562, 5602, 5602] dup_rows_per_dst=[2644, 2632, 2740, 2524, 2616, 2552, 2494, 2620] combine_rows_per_src=[5648, 5632, 5608, 5500, 5560, 5562, 5602, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44672 payload_rows_remote=39112 dup_rows=20864 dup_rows_remote=18280 dedup_ratio=0.318359 router_tokens_per_expert=[525, 519, 515, 494, 507, 574, 477, 538, 471, 531, 486, 495, 463, 520, 506, 549, 473, 505, 528, 499, 530, 473, 485, 504, 528, 479, 516, 491, 533, 529, 502, 570, 493, 524, 525, 510, 489, 488, 530, 513, 501, 509, 501, 529, 497, 513, 523, 540, 534, 520, 509, 509, 499, 493, 539, 534, 507, 518, 541, 514, 521, 545, 473, 512] payload_rows_per_dst=[5654, 5498, 5538, 5606, 5560, 5646, 5586, 5584] dup_rows_per_dst=[2644, 2544, 2456, 2690, 2584, 2580, 2688, 2678] combine_rows_per_src=[5654, 5498, 5538, 5606, 5560, 5646, 5586, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278003712 dispatch_meta_bytes=228928 combine_payload_bytes=278003712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278232640 ingress_bytes=388104192 link_bytes=388104192 token_bytes=556236352 weight_movement_bytes=110100480 total_nvlink_bytes=666336832 weight_to_token_ratio=0.197938 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749770221e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279810048 dispatch_meta_bytes=229872 combine_payload_bytes=279810048 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280039920 ingress_bytes=500011008 link_bytes=500011008 token_bytes=559849968 weight_movement_bytes=220200960 total_nvlink_bytes=780050928 weight_to_token_ratio=0.393321 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.696013806e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39062 dup_rows=20968 dup_rows_remote=18332 dedup_ratio=0.319946 router_tokens_per_expert=[472, 523, 527, 487, 531, 456, 556, 495, 522, 533, 484, 503, 535, 521, 503, 494, 502, 525, 507, 534, 514, 516, 464, 509, 502, 537, 515, 491, 518, 502, 520, 533, 512, 500, 525, 489, 509, 535, 538, 514, 495, 524, 527, 538, 516, 500, 510, 478, 484, 540, 537, 524, 545, 504, 500, 493, 491, 548, 520, 535, 514, 493, 517, 482] payload_rows_per_dst=[5556, 5598, 5506, 5590, 5564, 5566, 5648, 5540] dup_rows_per_dst=[2538, 2592, 2636, 2646, 2680, 2610, 2606, 2660] combine_rows_per_src=[5556, 5598, 5506, 5590, 5564, 5566, 5648, 5540] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278448128 dispatch_meta_bytes=229344 combine_payload_bytes=278448128 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278677472 ingress_bytes=608749568 link_bytes=608749568 token_bytes=557125600 weight_movement_bytes=330301440 total_nvlink_bytes=887427040 weight_to_token_ratio=0.592867 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058715621e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278003712 dispatch_meta_bytes=228928 combine_payload_bytes=278003712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278232640 ingress_bytes=388104192 link_bytes=388104192 token_bytes=556236352 weight_movement_bytes=110100480 total_nvlink_bytes=666336832 weight_to_token_ratio=0.197938 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749770221e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279064576 dispatch_meta_bytes=229216 combine_payload_bytes=279064576 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279293792 ingress_bytes=609366016 link_bytes=609366016 token_bytes=558358368 weight_movement_bytes=330301440 total_nvlink_bytes=888659808 weight_to_token_ratio=0.591558 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059787725e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=228976 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280254064 ingress_bytes=390125568 link_bytes=390125568 token_bytes=560279152 weight_movement_bytes=110100480 total_nvlink_bytes=670379632 weight_to_token_ratio=0.196510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784925274e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280483840 dispatch_meta_bytes=229408 combine_payload_bytes=280483840 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280713248 ingress_bytes=500684800 link_bytes=500684800 token_bytes=561197088 weight_movement_bytes=220200960 total_nvlink_bytes=781398048 weight_to_token_ratio=0.392377 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.707732157e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279996416 dispatch_meta_bytes=229576 combine_payload_bytes=279996416 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280225992 ingress_bytes=500197376 link_bytes=500197376 token_bytes=560222408 weight_movement_bytes=220200960 total_nvlink_bytes=780423368 weight_to_token_ratio=0.393060 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.699255052e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38956 dup_rows=20966 dup_rows_remote=18374 dedup_ratio=0.319916 router_tokens_per_expert=[536, 528, 512, 485, 496, 515, 541, 530, 495, 529, 564, 460, 496, 491, 522, 539, 511, 503, 480, 536, 507, 520, 518, 478, 518, 535, 535, 495, 507, 526, 522, 535, 511, 519, 462, 581, 478, 540, 497, 515, 542, 481, 508, 469, 553, 527, 520, 514, 508, 504, 503, 522, 480, 535, 485, 514, 532, 484, 502, 502, 478, 514, 526, 497] payload_rows_per_dst=[5596, 5568, 5502, 5660, 5614, 5646, 5554, 5430] dup_rows_per_dst=[2690, 2624, 2604, 2686, 2592, 2582, 2548, 2640] combine_rows_per_src=[5596, 5568, 5502, 5660, 5614, 5646, 5554, 5430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280354816 dispatch_meta_bytes=229568 combine_payload_bytes=280354816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280584384 ingress_bytes=390455296 link_bytes=390455296 token_bytes=560939200 weight_movement_bytes=110100480 total_nvlink_bytes=671039680 weight_to_token_ratio=0.196279 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790659786e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44638 payload_rows_remote=39228 dup_rows=20898 dup_rows_remote=18350 dedup_ratio=0.318878 router_tokens_per_expert=[488, 514, 476, 514, 532, 526, 510, 510, 538, 517, 502, 573, 456, 566, 520, 533, 498, 529, 511, 560, 530, 529, 506, 503, 510, 506, 518, 502, 523, 530, 540, 500, 517, 503, 530, 508, 473, 503, 498, 533, 467, 489, 483, 517, 522, 492, 514, 495, 521, 494, 504, 517, 496, 498, 542, 521, 512, 522, 517, 501, 508, 498, 485, 518] payload_rows_per_dst=[5550, 5664, 5656, 5662, 5572, 5410, 5604, 5520] dup_rows_per_dst=[2590, 2746, 2676, 2596, 2558, 2548, 2582, 2602] combine_rows_per_src=[5550, 5664, 5656, 5662, 5572, 5410, 5604, 5520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44356 payload_rows_remote=38810 dup_rows=21180 dup_rows_remote=18440 dedup_ratio=0.323181 router_tokens_per_expert=[492, 476, 525, 477, 519, 475, 545, 549, 547, 525, 521, 536, 507, 522, 532, 527, 527, 546, 506, 518, 531, 500, 519, 498, 523, 491, 517, 515, 520, 517, 549, 479, 511, 549, 510, 517, 510, 518, 455, 490, 518, 493, 508, 537, 536, 491, 484, 502, 499, 511, 510, 476, 522, 533, 527, 479, 507, 490, 512, 527, 499, 507, 525, 484] payload_rows_per_dst=[5478, 5548, 5628, 5486, 5500, 5500, 5670, 5546] dup_rows_per_dst=[2638, 2510, 2662, 2552, 2574, 2684, 2820, 2740] combine_rows_per_src=[5478, 5548, 5628, 5486, 5500, 5500, 5670, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229320 combine_payload_bytes=279236608 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=279465928 ingress_bytes=719638528 link_bytes=719638528 token_bytes=558702536 weight_movement_bytes=440401920 total_nvlink_bytes=999104456 weight_to_token_ratio=0.788258 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251569760e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279996416 dispatch_meta_bytes=229576 combine_payload_bytes=279996416 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280225992 ingress_bytes=500197376 link_bytes=500197376 token_bytes=560222408 weight_movement_bytes=220200960 total_nvlink_bytes=780423368 weight_to_token_ratio=0.393060 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.699255052e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44538 payload_rows_remote=38890 dup_rows=20998 dup_rows_remote=18470 dedup_ratio=0.320404 router_tokens_per_expert=[505, 496, 514, 500, 531, 528, 523, 483, 482, 532, 513, 508, 483, 490, 541, 523, 535, 506, 543, 480, 513, 476, 508, 567, 497, 514, 515, 520, 483, 504, 511, 552, 526, 516, 493, 524, 535, 545, 490, 498, 499, 527, 518, 534, 516, 513, 507, 475, 506, 516, 484, 506, 506, 567, 517, 486, 500, 487, 535, 544, 506, 504, 500, 512] payload_rows_per_dst=[5536, 5580, 5576, 5602, 5526, 5510, 5648, 5560] dup_rows_per_dst=[2624, 2564, 2680, 2590, 2728, 2668, 2528, 2616] combine_rows_per_src=[5536, 5580, 5576, 5602, 5526, 5510, 5648, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=281186304 dispatch_meta_bytes=230312 combine_payload_bytes=281186304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281416616 ingress_bytes=391286784 link_bytes=391286784 token_bytes=562602920 weight_movement_bytes=110100480 total_nvlink_bytes=672703400 weight_to_token_ratio=0.195698 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.805120730e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44708 payload_rows_remote=39130 dup_rows=20828 dup_rows_remote=18222 dedup_ratio=0.317810 router_tokens_per_expert=[477, 480, 504, 535, 539, 514, 518, 503, 513, 471, 535, 529, 526, 493, 516, 509, 467, 512, 486, 516, 501, 511, 482, 500, 509, 511, 542, 529, 491, 496, 491, 495, 509, 524, 544, 557, 503, 497, 509, 543, 491, 502, 525, 537, 536, 538, 536, 536, 522, 508, 524, 525, 503, 498, 478, 524, 483, 529, 515, 522, 496, 519, 529, 505] payload_rows_per_dst=[5576, 5578, 5500, 5550, 5706, 5592, 5562, 5644] dup_rows_per_dst=[2564, 2606, 2450, 2578, 2666, 2810, 2602, 2552] combine_rows_per_src=[5576, 5578, 5500, 5550, 5706, 5592, 5562, 5644] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44566 payload_rows_remote=39044 dup_rows=20970 dup_rows_remote=18326 dedup_ratio=0.319977 router_tokens_per_expert=[467, 514, 489, 536, 574, 512, 504, 505, 507, 542, 517, 495, 494, 517, 495, 521, 461, 493, 507, 538, 538, 481, 500, 536, 495, 502, 505, 499, 538, 525, 500, 519, 538, 511, 491, 524, 526, 479, 497, 500, 516, 515, 531, 490, 509, 512, 489, 537, 487, 518, 516, 469, 518, 532, 502, 544, 528, 536, 475, 539, 492, 527, 577, 517] payload_rows_per_dst=[5602, 5594, 5492, 5522, 5504, 5616, 5614, 5622] dup_rows_per_dst=[2600, 2582, 2616, 2644, 2628, 2582, 2558, 2760] combine_rows_per_src=[5602, 5594, 5492, 5522, 5504, 5616, 5614, 5622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44356 payload_rows_remote=38810 dup_rows=21180 dup_rows_remote=18440 dedup_ratio=0.323181 router_tokens_per_expert=[492, 476, 525, 477, 519, 475, 545, 549, 547, 525, 521, 536, 507, 522, 532, 527, 527, 546, 506, 518, 531, 500, 519, 498, 523, 491, 517, 515, 520, 517, 549, 479, 511, 549, 510, 517, 510, 518, 455, 490, 518, 493, 508, 537, 536, 491, 484, 502, 499, 511, 510, 476, 522, 533, 527, 479, 507, 490, 512, 527, 499, 507, 525, 484] payload_rows_per_dst=[5478, 5548, 5628, 5486, 5500, 5500, 5670, 5546] dup_rows_per_dst=[2638, 2510, 2662, 2552, 2574, 2684, 2820, 2740] combine_rows_per_src=[5478, 5548, 5628, 5486, 5500, 5500, 5670, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44538 payload_rows_remote=38890 dup_rows=20998 dup_rows_remote=18470 dedup_ratio=0.320404 router_tokens_per_expert=[505, 496, 514, 500, 531, 528, 523, 483, 482, 532, 513, 508, 483, 490, 541, 523, 535, 506, 543, 480, 513, 476, 508, 567, 497, 514, 515, 520, 483, 504, 511, 552, 526, 516, 493, 524, 535, 545, 490, 498, 499, 527, 518, 534, 516, 513, 507, 475, 506, 516, 484, 506, 506, 567, 517, 486, 500, 487, 535, 544, 506, 504, 500, 512] payload_rows_per_dst=[5536, 5580, 5576, 5602, 5526, 5510, 5648, 5560] dup_rows_per_dst=[2624, 2564, 2680, 2590, 2728, 2668, 2528, 2616] combine_rows_per_src=[5536, 5580, 5576, 5602, 5526, 5510, 5648, 5560] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38940 dup_rows=21064 dup_rows_remote=18286 dedup_ratio=0.321411 router_tokens_per_expert=[552, 504, 476, 570, 508, 472, 492, 491, 516, 491, 533, 501, 486, 517, 541, 506, 548, 518, 513, 505, 556, 513, 519, 518, 440, 501, 507, 506, 508, 559, 504, 519, 510, 544, 520, 519, 543, 555, 504, 527, 493, 488, 480, 524, 525, 520, 524, 488, 511, 510, 493, 488, 556, 509, 472, 472, 546, 506, 543, 489, 526, 504, 480, 509] payload_rows_per_dst=[5532, 5636, 5696, 5460, 5638, 5522, 5436, 5552] dup_rows_per_dst=[2778, 2636, 2686, 2558, 2604, 2562, 2586, 2654] combine_rows_per_src=[5532, 5636, 5696, 5460, 5638, 5522, 5436, 5552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38832 dup_rows=21142 dup_rows_remote=18462 dedup_ratio=0.322601 router_tokens_per_expert=[524, 529, 525, 511, 557, 537, 566, 520, 542, 525, 513, 462, 530, 519, 547, 483, 506, 523, 508, 525, 477, 505, 530, 514, 521, 507, 518, 523, 473, 516, 540, 503, 468, 539, 501, 509, 520, 517, 509, 514, 499, 491, 517, 514, 518, 530, 475, 470, 534, 481, 569, 486, 487, 507, 508, 491, 481, 504, 483, 511, 512, 527, 513, 504] payload_rows_per_dst=[5772, 5562, 5516, 5508, 5482, 5540, 5544, 5470] dup_rows_per_dst=[2766, 2680, 2660, 2694, 2672, 2488, 2582, 2600] combine_rows_per_src=[5772, 5562, 5516, 5508, 5482, 5540, 5544, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38956 dup_rows=20966 dup_rows_remote=18374 dedup_ratio=0.319916 router_tokens_per_expert=[536, 528, 512, 485, 496, 515, 541, 530, 495, 529, 564, 460, 496, 491, 522, 539, 511, 503, 480, 536, 507, 520, 518, 478, 518, 535, 535, 495, 507, 526, 522, 535, 511, 519, 462, 581, 478, 540, 497, 515, 542, 481, 508, 469, 553, 527, 520, 514, 508, 504, 503, 522, 480, 535, 485, 514, 532, 484, 502, 502, 478, 514, 526, 497] payload_rows_per_dst=[5596, 5568, 5502, 5660, 5614, 5646, 5554, 5430] dup_rows_per_dst=[2690, 2624, 2604, 2686, 2592, 2582, 2548, 2640] combine_rows_per_src=[5596, 5568, 5502, 5660, 5614, 5646, 5554, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280483840 dispatch_meta_bytes=229408 combine_payload_bytes=280483840 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280713248 ingress_bytes=500684800 link_bytes=500684800 token_bytes=561197088 weight_movement_bytes=220200960 total_nvlink_bytes=781398048 weight_to_token_ratio=0.392377 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.707732157e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38952 dup_rows=21050 dup_rows_remote=18472 dedup_ratio=0.321198 router_tokens_per_expert=[535, 504, 532, 515, 506, 498, 512, 489, 503, 490, 525, 484, 524, 518, 521, 485, 538, 550, 471, 502, 507, 519, 473, 496, 499, 532, 523, 540, 519, 509, 515, 558, 518, 524, 499, 515, 473, 498, 515, 521, 506, 522, 538, 514, 498, 486, 559, 529, 532, 523, 535, 507, 526, 517, 522, 450, 502, 502, 524, 489, 487, 523, 506, 516] payload_rows_per_dst=[5542, 5524, 5534, 5682, 5580, 5596, 5534, 5494] dup_rows_per_dst=[2640, 2576, 2578, 2708, 2546, 2708, 2690, 2604] combine_rows_per_src=[5542, 5524, 5534, 5682, 5580, 5596, 5534, 5494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=281186304 dispatch_meta_bytes=230312 combine_payload_bytes=281186304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281416616 ingress_bytes=391286784 link_bytes=391286784 token_bytes=562602920 weight_movement_bytes=110100480 total_nvlink_bytes=672703400 weight_to_token_ratio=0.195698 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.805120730e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279867392 dispatch_meta_bytes=229480 combine_payload_bytes=279867392 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280096872 ingress_bytes=610168832 link_bytes=610168832 token_bytes=559964264 weight_movement_bytes=330301440 total_nvlink_bytes=890265704 weight_to_token_ratio=0.589862 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061183954e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278190080 dispatch_meta_bytes=229000 combine_payload_bytes=278190080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278419080 ingress_bytes=388290560 link_bytes=388290560 token_bytes=556609160 weight_movement_bytes=110100480 total_nvlink_bytes=666709640 weight_to_token_ratio=0.197806 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753011467e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278190080 dispatch_meta_bytes=229000 combine_payload_bytes=278190080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278419080 ingress_bytes=388290560 link_bytes=388290560 token_bytes=556609160 weight_movement_bytes=110100480 total_nvlink_bytes=666709640 weight_to_token_ratio=0.197806 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753011467e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44616 payload_rows_remote=39140 dup_rows=20920 dup_rows_remote=18262 dedup_ratio=0.319214 router_tokens_per_expert=[524, 483, 509, 498, 514, 522, 497, 518, 520, 507, 532, 473, 538, 527, 531, 528, 494, 521, 492, 513, 500, 502, 493, 518, 531, 519, 537, 530, 496, 481, 537, 544, 519, 508, 519, 525, 466, 490, 542, 512, 476, 519, 502, 529, 483, 510, 548, 500, 507, 537, 492, 501, 524, 491, 517, 511, 521, 523, 479, 506, 518, 495, 547, 522] payload_rows_per_dst=[5604, 5632, 5464, 5642, 5606, 5476, 5622, 5570] dup_rows_per_dst=[2526, 2680, 2602, 2708, 2556, 2658, 2538, 2652] combine_rows_per_src=[5604, 5632, 5464, 5642, 5606, 5476, 5622, 5570] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229440 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278992960 ingress_bytes=388864000 link_bytes=388864000 token_bytes=557756480 weight_movement_bytes=110100480 total_nvlink_bytes=667856960 weight_to_token_ratio=0.197399 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762984531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279867392 dispatch_meta_bytes=229480 combine_payload_bytes=279867392 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280096872 ingress_bytes=610168832 link_bytes=610168832 token_bytes=559964264 weight_movement_bytes=330301440 total_nvlink_bytes=890265704 weight_to_token_ratio=0.589862 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061183954e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229320 combine_payload_bytes=279236608 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=279465928 ingress_bytes=719638528 link_bytes=719638528 token_bytes=558702536 weight_movement_bytes=440401920 total_nvlink_bytes=999104456 weight_to_token_ratio=0.788258 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251569760e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=228904 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279350824 ingress_bytes=389222400 link_bytes=389222400 token_bytes=558472744 weight_movement_bytes=110100480 total_nvlink_bytes=668573224 weight_to_token_ratio=0.197146 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769217697e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278347776 dispatch_meta_bytes=229176 combine_payload_bytes=278347776 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=278576952 ingress_bytes=828850176 link_bytes=828850176 token_bytes=556924728 weight_movement_bytes=550502400 total_nvlink_bytes=1107427128 weight_to_token_ratio=0.988468 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.441506778e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229696 combine_payload_bytes=279207936 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279437632 ingress_bytes=389308416 link_bytes=389308416 token_bytes=558645568 weight_movement_bytes=110100480 total_nvlink_bytes=668746048 weight_to_token_ratio=0.197085 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770713657e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44504 payload_rows_remote=38954 dup_rows=21032 dup_rows_remote=18312 dedup_ratio=0.320923 router_tokens_per_expert=[496, 509, 496, 501, 517, 496, 498, 503, 489, 506, 529, 525, 528, 528, 533, 503, 545, 564, 510, 504, 474, 516, 477, 487, 488, 501, 568, 490, 510, 504, 489, 500, 520, 542, 466, 494, 525, 531, 534, 523, 519, 528, 489, 518, 516, 495, 498, 529, 515, 544, 502, 501, 532, 516, 509, 521, 538, 527, 484, 531, 527, 494, 499, 517] payload_rows_per_dst=[5466, 5694, 5566, 5502, 5550, 5572, 5592, 5562] dup_rows_per_dst=[2566, 2588, 2588, 2598, 2720, 2612, 2688, 2672] combine_rows_per_src=[5466, 5694, 5566, 5502, 5550, 5572, 5592, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44616 payload_rows_remote=39140 dup_rows=20920 dup_rows_remote=18262 dedup_ratio=0.319214 router_tokens_per_expert=[524, 483, 509, 498, 514, 522, 497, 518, 520, 507, 532, 473, 538, 527, 531, 528, 494, 521, 492, 513, 500, 502, 493, 518, 531, 519, 537, 530, 496, 481, 537, 544, 519, 508, 519, 525, 466, 490, 542, 512, 476, 519, 502, 529, 483, 510, 548, 500, 507, 537, 492, 501, 524, 491, 517, 511, 521, 523, 479, 506, 518, 495, 547, 522] payload_rows_per_dst=[5604, 5632, 5464, 5642, 5606, 5476, 5622, 5570] dup_rows_per_dst=[2526, 2680, 2602, 2708, 2556, 2658, 2538, 2652] combine_rows_per_src=[5604, 5632, 5464, 5642, 5606, 5476, 5622, 5570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229064 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279451336 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558673608 weight_movement_bytes=110100480 total_nvlink_bytes=668774088 weight_to_token_ratio=0.197075 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38786 dup_rows=21124 dup_rows_remote=18364 dedup_ratio=0.322327 router_tokens_per_expert=[505, 537, 459, 516, 463, 502, 480, 517, 531, 489, 510, 532, 523, 535, 504, 554, 505, 531, 497, 490, 502, 486, 499, 496, 496, 491, 511, 515, 537, 522, 520, 511, 516, 513, 529, 538, 512, 507, 548, 517, 499, 507, 537, 534, 469, 496, 493, 505, 517, 530, 543, 525, 500, 517, 515, 531, 493, 495, 541, 516, 496, 516, 517, 530] payload_rows_per_dst=[5462, 5376, 5494, 5532, 5640, 5644, 5638, 5626] dup_rows_per_dst=[2496, 2396, 2590, 2674, 2720, 2770, 2718, 2760] combine_rows_per_src=[5462, 5376, 5494, 5532, 5640, 5644, 5638, 5626] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38952 dup_rows=21046 dup_rows_remote=18430 dedup_ratio=0.321136 router_tokens_per_expert=[501, 489, 550, 511, 537, 507, 495, 507, 492, 529, 543, 512, 523, 501, 515, 525, 504, 502, 539, 512, 522, 530, 517, 515, 525, 489, 493, 487, 518, 473, 533, 540, 515, 544, 488, 494, 521, 475, 511, 514, 521, 535, 497, 503, 503, 539, 500, 563, 525, 530, 481, 514, 505, 502, 515, 505, 520, 493, 494, 504, 500, 508, 488, 525] payload_rows_per_dst=[5636, 5622, 5594, 5458, 5490, 5620, 5538, 5532] dup_rows_per_dst=[2558, 2658, 2688, 2658, 2634, 2702, 2616, 2532] combine_rows_per_src=[5636, 5622, 5594, 5458, 5490, 5620, 5538, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280555520 dispatch_meta_bytes=229608 combine_payload_bytes=280555520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280785128 ingress_bytes=390656000 link_bytes=390656000 token_bytes=561340648 weight_movement_bytes=110100480 total_nvlink_bytes=671441128 weight_to_token_ratio=0.196138 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794150359e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38832 dup_rows=21142 dup_rows_remote=18462 dedup_ratio=0.322601 router_tokens_per_expert=[524, 529, 525, 511, 557, 537, 566, 520, 542, 525, 513, 462, 530, 519, 547, 483, 506, 523, 508, 525, 477, 505, 530, 514, 521, 507, 518, 523, 473, 516, 540, 503, 468, 539, 501, 509, 520, 517, 509, 514, 499, 491, 517, 514, 518, 530, 475, 470, 534, 481, 569, 486, 487, 507, 508, 491, 481, 504, 483, 511, 512, 527, 513, 504] payload_rows_per_dst=[5772, 5562, 5516, 5508, 5482, 5540, 5544, 5470] dup_rows_per_dst=[2766, 2680, 2660, 2694, 2672, 2488, 2582, 2600] combine_rows_per_src=[5772, 5562, 5516, 5508, 5482, 5540, 5544, 5470] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44660 payload_rows_remote=39104 dup_rows=20876 dup_rows_remote=18330 dedup_ratio=0.318542 router_tokens_per_expert=[518, 490, 504, 511, 505, 513, 515, 515, 492, 486, 559, 508, 516, 528, 529, 507, 500, 532, 511, 521, 524, 532, 515, 534, 500, 497, 490, 526, 522, 491, 518, 507, 523, 557, 538, 499, 514, 530, 501, 531, 498, 468, 511, 498, 514, 525, 513, 520, 513, 525, 504, 482, 523, 533, 516, 471, 489, 484, 532, 505, 490, 533, 517, 495] payload_rows_per_dst=[5558, 5610, 5634, 5556, 5642, 5510, 5574, 5576] dup_rows_per_dst=[2584, 2640, 2704, 2546, 2744, 2584, 2560, 2514] combine_rows_per_src=[5558, 5610, 5634, 5556, 5642, 5510, 5574, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38786 dup_rows=21124 dup_rows_remote=18364 dedup_ratio=0.322327 router_tokens_per_expert=[505, 537, 459, 516, 463, 502, 480, 517, 531, 489, 510, 532, 523, 535, 504, 554, 505, 531, 497, 490, 502, 486, 499, 496, 496, 491, 511, 515, 537, 522, 520, 511, 516, 513, 529, 538, 512, 507, 548, 517, 499, 507, 537, 534, 469, 496, 493, 505, 517, 530, 543, 525, 500, 517, 515, 531, 493, 495, 541, 516, 496, 516, 517, 530] payload_rows_per_dst=[5462, 5376, 5494, 5532, 5640, 5644, 5638, 5626] dup_rows_per_dst=[2496, 2396, 2590, 2674, 2720, 2770, 2718, 2760] combine_rows_per_src=[5462, 5376, 5494, 5532, 5640, 5644, 5638, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=38914 dup_rows=21006 dup_rows_remote=18362 dedup_ratio=0.320526 router_tokens_per_expert=[561, 515, 501, 504, 490, 487, 513, 492, 531, 527, 525, 505, 492, 529, 507, 559, 515, 460, 528, 516, 515, 490, 513, 508, 532, 507, 521, 486, 522, 517, 532, 537, 452, 496, 525, 511, 517, 495, 497, 516, 482, 502, 476, 543, 523, 503, 526, 534, 538, 496, 493, 529, 505, 490, 511, 516, 530, 519, 533, 520, 519, 515, 473, 546] payload_rows_per_dst=[5616, 5578, 5552, 5598, 5492, 5606, 5516, 5572] dup_rows_per_dst=[2644, 2658, 2538, 2786, 2534, 2572, 2536, 2738] combine_rows_per_src=[5616, 5578, 5552, 5598, 5492, 5606, 5516, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280555520 dispatch_meta_bytes=229608 combine_payload_bytes=280555520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280785128 ingress_bytes=390656000 link_bytes=390656000 token_bytes=561340648 weight_movement_bytes=110100480 total_nvlink_bytes=671441128 weight_to_token_ratio=0.196138 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.794150359e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38998 dup_rows=21014 dup_rows_remote=18316 dedup_ratio=0.320648 router_tokens_per_expert=[508, 517, 495, 504, 479, 524, 506, 531, 504, 509, 507, 509, 539, 512, 494, 537, 525, 495, 511, 465, 451, 514, 540, 555, 543, 538, 517, 510, 511, 484, 499, 527, 515, 524, 500, 506, 490, 527, 502, 506, 492, 506, 493, 540, 520, 513, 506, 545, 494, 508, 520, 504, 532, 510, 475, 505, 508, 506, 528, 505, 505, 536, 531, 556] payload_rows_per_dst=[5502, 5524, 5592, 5570, 5596, 5582, 5516, 5640] dup_rows_per_dst=[2626, 2698, 2520, 2688, 2544, 2648, 2580, 2710] combine_rows_per_src=[5502, 5524, 5592, 5570, 5596, 5582, 5516, 5640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44628 payload_rows_remote=38966 dup_rows=20908 dup_rows_remote=18224 dedup_ratio=0.319031 router_tokens_per_expert=[514, 540, 545, 506, 553, 498, 466, 519, 510, 523, 495, 492, 530, 512, 547, 472, 543, 496, 504, 522, 501, 513, 518, 576, 513, 511, 519, 491, 513, 498, 478, 515, 495, 539, 537, 516, 489, 488, 467, 496, 497, 517, 533, 506, 505, 517, 529, 487, 505, 540, 509, 498, 498, 523, 537, 527, 443, 505, 519, 508, 513, 526, 535, 531] payload_rows_per_dst=[5628, 5520, 5662, 5570, 5578, 5472, 5624, 5574] dup_rows_per_dst=[2654, 2642, 2684, 2506, 2476, 2710, 2650, 2586] combine_rows_per_src=[5628, 5520, 5662, 5570, 5578, 5472, 5624, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38952 dup_rows=21046 dup_rows_remote=18430 dedup_ratio=0.321136 router_tokens_per_expert=[501, 489, 550, 511, 537, 507, 495, 507, 492, 529, 543, 512, 523, 501, 515, 525, 504, 502, 539, 512, 522, 530, 517, 515, 525, 489, 493, 487, 518, 473, 533, 540, 515, 544, 488, 494, 521, 475, 511, 514, 521, 535, 497, 503, 503, 539, 500, 563, 525, 530, 481, 514, 505, 502, 515, 505, 520, 493, 494, 504, 500, 508, 488, 525] payload_rows_per_dst=[5636, 5622, 5594, 5458, 5490, 5620, 5538, 5532] dup_rows_per_dst=[2558, 2658, 2688, 2658, 2634, 2702, 2616, 2532] combine_rows_per_src=[5636, 5622, 5594, 5458, 5490, 5620, 5538, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278347776 dispatch_meta_bytes=229176 combine_payload_bytes=278347776 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=278576952 ingress_bytes=828850176 link_bytes=828850176 token_bytes=556924728 weight_movement_bytes=550502400 total_nvlink_bytes=1107427128 weight_to_token_ratio=0.988468 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.441506778e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280297472 dispatch_meta_bytes=229736 combine_payload_bytes=280297472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280527208 ingress_bytes=390397952 link_bytes=390397952 token_bytes=560824680 weight_movement_bytes=110100480 total_nvlink_bytes=670925160 weight_to_token_ratio=0.196319 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789662480e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278018048 dispatch_meta_bytes=228600 combine_payload_bytes=278018048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278246648 ingress_bytes=388118528 link_bytes=388118528 token_bytes=556264696 weight_movement_bytes=110100480 total_nvlink_bytes=666365176 weight_to_token_ratio=0.197928 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750019547e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44504 payload_rows_remote=38954 dup_rows=21032 dup_rows_remote=18312 dedup_ratio=0.320923 router_tokens_per_expert=[496, 509, 496, 501, 517, 496, 498, 503, 489, 506, 529, 525, 528, 528, 533, 503, 545, 564, 510, 504, 474, 516, 477, 487, 488, 501, 568, 490, 510, 504, 489, 500, 520, 542, 466, 494, 525, 531, 534, 523, 519, 528, 489, 518, 516, 495, 498, 529, 515, 544, 502, 501, 532, 516, 509, 521, 538, 527, 484, 531, 527, 494, 499, 517] payload_rows_per_dst=[5466, 5694, 5566, 5502, 5550, 5572, 5592, 5562] dup_rows_per_dst=[2566, 2588, 2588, 2598, 2720, 2612, 2688, 2672] combine_rows_per_src=[5466, 5694, 5566, 5502, 5550, 5572, 5592, 5562] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229528 combine_payload_bytes=279207936 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279437464 ingress_bytes=609509376 link_bytes=609509376 token_bytes=558645400 weight_movement_bytes=330301440 total_nvlink_bytes=888946840 weight_to_token_ratio=0.591254 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060037052e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44350 payload_rows_remote=38802 dup_rows=21186 dup_rows_remote=18478 dedup_ratio=0.323273 router_tokens_per_expert=[493, 531, 509, 527, 525, 519, 516, 524, 479, 526, 512, 515, 519, 535, 513, 504, 527, 493, 485, 521, 502, 530, 523, 518, 528, 492, 484, 507, 499, 511, 531, 481, 512, 518, 494, 503, 525, 525, 505, 546, 522, 511, 505, 526, 535, 502, 520, 495, 512, 519, 515, 483, 490, 535, 511, 508, 502, 521, 543, 493, 513, 508, 476, 516] payload_rows_per_dst=[5622, 5532, 5546, 5466, 5548, 5598, 5488, 5550] dup_rows_per_dst=[2666, 2674, 2652, 2600, 2708, 2634, 2658, 2594] combine_rows_per_src=[5622, 5532, 5546, 5466, 5548, 5598, 5488, 5550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280297472 dispatch_meta_bytes=229736 combine_payload_bytes=280297472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280527208 ingress_bytes=390397952 link_bytes=390397952 token_bytes=560824680 weight_movement_bytes=110100480 total_nvlink_bytes=670925160 weight_to_token_ratio=0.196319 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789662480e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=229104 combine_payload_bytes=278935552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279164656 ingress_bytes=389036032 link_bytes=389036032 token_bytes=558100208 weight_movement_bytes=110100480 total_nvlink_bytes=668200688 weight_to_token_ratio=0.197277 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765976451e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=229256 combine_payload_bytes=279537664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279766920 ingress_bytes=499738624 link_bytes=499738624 token_bytes=559304584 weight_movement_bytes=220200960 total_nvlink_bytes=779505544 weight_to_token_ratio=0.393705 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.691276600e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=228760 combine_payload_bytes=279308288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279537048 ingress_bytes=389408768 link_bytes=389408768 token_bytes=558845336 weight_movement_bytes=110100480 total_nvlink_bytes=668945816 weight_to_token_ratio=0.197014 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772458943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278132736 dispatch_meta_bytes=229120 combine_payload_bytes=278132736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278361856 ingress_bytes=388233216 link_bytes=388233216 token_bytes=556494592 weight_movement_bytes=110100480 total_nvlink_bytes=666595072 weight_to_token_ratio=0.197846 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752014160e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44628 payload_rows_remote=38966 dup_rows=20908 dup_rows_remote=18224 dedup_ratio=0.319031 router_tokens_per_expert=[514, 540, 545, 506, 553, 498, 466, 519, 510, 523, 495, 492, 530, 512, 547, 472, 543, 496, 504, 522, 501, 513, 518, 576, 513, 511, 519, 491, 513, 498, 478, 515, 495, 539, 537, 516, 489, 488, 467, 496, 497, 517, 533, 506, 505, 517, 529, 487, 505, 540, 509, 498, 498, 523, 537, 527, 443, 505, 519, 508, 513, 526, 535, 531] payload_rows_per_dst=[5628, 5520, 5662, 5570, 5578, 5472, 5624, 5574] dup_rows_per_dst=[2654, 2642, 2684, 2506, 2476, 2710, 2650, 2586] combine_rows_per_src=[5628, 5520, 5662, 5570, 5578, 5472, 5624, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44592 payload_rows_remote=39024 dup_rows=20944 dup_rows_remote=18382 dedup_ratio=0.319580 router_tokens_per_expert=[476, 527, 523, 539, 520, 532, 510, 535, 493, 502, 508, 513, 505, 563, 498, 523, 507, 500, 501, 538, 507, 534, 520, 528, 507, 515, 535, 538, 509, 485, 542, 555, 502, 517, 517, 515, 495, 494, 519, 506, 520, 504, 497, 514, 514, 505, 492, 519, 503, 506, 529, 499, 498, 480, 471, 498, 524, 489, 504, 549, 479, 492, 489, 540] payload_rows_per_dst=[5624, 5550, 5594, 5644, 5514, 5568, 5508, 5590] dup_rows_per_dst=[2700, 2660, 2676, 2728, 2616, 2562, 2460, 2542] combine_rows_per_src=[5624, 5550, 5594, 5644, 5514, 5568, 5508, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=229104 combine_payload_bytes=278935552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279164656 ingress_bytes=389036032 link_bytes=389036032 token_bytes=558100208 weight_movement_bytes=110100480 total_nvlink_bytes=668200688 weight_to_token_ratio=0.197277 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765976451e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=39076 dup_rows=20978 dup_rows_remote=18378 dedup_ratio=0.320099 router_tokens_per_expert=[531, 533, 561, 506, 520, 520, 519, 489, 528, 523, 519, 477, 502, 517, 501, 500, 508, 500, 525, 514, 502, 530, 497, 536, 518, 559, 511, 466, 510, 533, 508, 479, 518, 494, 469, 518, 536, 503, 510, 506, 516, 505, 520, 516, 551, 502, 537, 503, 515, 543, 508, 525, 473, 504, 480, 533, 519, 474, 517, 497, 485, 522, 498, 529] payload_rows_per_dst=[5568, 5384, 5544, 5416, 5740, 5748, 5676, 5482] dup_rows_per_dst=[2660, 2426, 2568, 2290, 2830, 2876, 2728, 2600] combine_rows_per_src=[5568, 5384, 5544, 5416, 5740, 5748, 5676, 5482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38934 dup_rows=21100 dup_rows_remote=18374 dedup_ratio=0.321960 router_tokens_per_expert=[521, 513, 478, 564, 511, 500, 509, 560, 512, 513, 538, 498, 488, 547, 530, 524, 554, 525, 525, 514, 506, 515, 515, 528, 512, 479, 531, 527, 492, 540, 521, 496, 498, 493, 485, 522, 508, 482, 483, 502, 532, 507, 543, 477, 519, 496, 473, 498, 508, 488, 498, 543, 513, 517, 536, 511, 545, 520, 486, 487, 501, 491, 508, 512] payload_rows_per_dst=[5644, 5636, 5608, 5540, 5432, 5570, 5502, 5504] dup_rows_per_dst=[2668, 2664, 2756, 2656, 2514, 2520, 2726, 2596] combine_rows_per_src=[5644, 5636, 5608, 5540, 5432, 5570, 5502, 5504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38894 dup_rows=21002 dup_rows_remote=18244 dedup_ratio=0.320465 router_tokens_per_expert=[519, 517, 541, 501, 537, 470, 497, 524, 475, 523, 483, 545, 490, 548, 504, 521, 495, 487, 509, 535, 545, 494, 520, 528, 528, 508, 559, 490, 496, 557, 543, 518, 517, 492, 467, 500, 513, 524, 531, 512, 510, 539, 478, 488, 507, 486, 523, 503, 485, 514, 504, 507, 515, 540, 496, 504, 514, 531, 530, 497, 521, 504, 499, 510] payload_rows_per_dst=[5572, 5598, 5602, 5640, 5506, 5526, 5556, 5534] dup_rows_per_dst=[2640, 2580, 2624, 2758, 2606, 2542, 2574, 2678] combine_rows_per_src=[5572, 5598, 5602, 5640, 5506, 5526, 5556, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229528 combine_payload_bytes=279207936 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279437464 ingress_bytes=609509376 link_bytes=609509376 token_bytes=558645400 weight_movement_bytes=330301440 total_nvlink_bytes=888946840 weight_to_token_ratio=0.591254 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060037052e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229064 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279451336 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558673608 weight_movement_bytes=110100480 total_nvlink_bytes=668774088 weight_to_token_ratio=0.197075 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=228760 combine_payload_bytes=279308288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279537048 ingress_bytes=389408768 link_bytes=389408768 token_bytes=558845336 weight_movement_bytes=110100480 total_nvlink_bytes=668945816 weight_to_token_ratio=0.197014 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772458943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279724032 dispatch_meta_bytes=229624 combine_payload_bytes=279724032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279953656 ingress_bytes=389824512 link_bytes=389824512 token_bytes=559677688 weight_movement_bytes=110100480 total_nvlink_bytes=669778168 weight_to_token_ratio=0.196721 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779689415e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38998 dup_rows=21014 dup_rows_remote=18316 dedup_ratio=0.320648 router_tokens_per_expert=[508, 517, 495, 504, 479, 524, 506, 531, 504, 509, 507, 509, 539, 512, 494, 537, 525, 495, 511, 465, 451, 514, 540, 555, 543, 538, 517, 510, 511, 484, 499, 527, 515, 524, 500, 506, 490, 527, 502, 506, 492, 506, 493, 540, 520, 513, 506, 545, 494, 508, 520, 504, 532, 510, 475, 505, 508, 506, 528, 505, 505, 536, 531, 556] payload_rows_per_dst=[5502, 5524, 5592, 5570, 5596, 5582, 5516, 5640] dup_rows_per_dst=[2626, 2698, 2520, 2688, 2544, 2648, 2580, 2710] combine_rows_per_src=[5502, 5524, 5592, 5570, 5596, 5582, 5516, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38894 dup_rows=21002 dup_rows_remote=18244 dedup_ratio=0.320465 router_tokens_per_expert=[519, 517, 541, 501, 537, 470, 497, 524, 475, 523, 483, 545, 490, 548, 504, 521, 495, 487, 509, 535, 545, 494, 520, 528, 528, 508, 559, 490, 496, 557, 543, 518, 517, 492, 467, 500, 513, 524, 531, 512, 510, 539, 478, 488, 507, 486, 523, 503, 485, 514, 504, 507, 515, 540, 496, 504, 514, 531, 530, 497, 521, 504, 499, 510] payload_rows_per_dst=[5572, 5598, 5602, 5640, 5506, 5526, 5556, 5534] dup_rows_per_dst=[2640, 2580, 2624, 2758, 2606, 2542, 2574, 2678] combine_rows_per_src=[5572, 5598, 5602, 5640, 5506, 5526, 5556, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=39076 dup_rows=20978 dup_rows_remote=18378 dedup_ratio=0.320099 router_tokens_per_expert=[531, 533, 561, 506, 520, 520, 519, 489, 528, 523, 519, 477, 502, 517, 501, 500, 508, 500, 525, 514, 502, 530, 497, 536, 518, 559, 511, 466, 510, 533, 508, 479, 518, 494, 469, 518, 536, 503, 510, 506, 516, 505, 520, 516, 551, 502, 537, 503, 515, 543, 508, 525, 473, 504, 480, 533, 519, 474, 517, 497, 485, 522, 498, 529] payload_rows_per_dst=[5568, 5384, 5544, 5416, 5740, 5748, 5676, 5482] dup_rows_per_dst=[2660, 2426, 2568, 2290, 2830, 2876, 2728, 2600] combine_rows_per_src=[5568, 5384, 5544, 5416, 5740, 5748, 5676, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38848 dup_rows=21144 dup_rows_remote=18640 dedup_ratio=0.322632 router_tokens_per_expert=[466, 517, 508, 479, 501, 497, 520, 510, 545, 509, 499, 503, 501, 517, 501, 515, 526, 502, 483, 486, 516, 497, 513, 515, 513, 528, 541, 512, 520, 512, 550, 501, 450, 535, 555, 503, 538, 514, 516, 528, 510, 509, 460, 561, 487, 465, 510, 518, 494, 536, 533, 496, 538, 512, 552, 496, 506, 528, 545, 501, 538, 516, 495, 520] payload_rows_per_dst=[5544, 5556, 5500, 5618, 5628, 5440, 5580, 5526] dup_rows_per_dst=[2504, 2704, 2576, 2668, 2650, 2600, 2734, 2708] combine_rows_per_src=[5544, 5556, 5500, 5618, 5628, 5440, 5580, 5526] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44572 payload_rows_remote=39082 dup_rows=20964 dup_rows_remote=18412 dedup_ratio=0.319885 router_tokens_per_expert=[537, 518, 539, 526, 499, 522, 499, 502, 492, 512, 493, 491, 524, 479, 529, 501, 521, 497, 525, 528, 531, 501, 494, 450, 498, 479, 483, 509, 485, 519, 505, 484, 529, 533, 556, 500, 533, 494, 585, 537, 499, 520, 515, 519, 504, 514, 524, 507, 511, 508, 531, 490, 521, 532, 528, 513, 520, 491, 524, 519, 518, 501, 528, 492] payload_rows_per_dst=[5576, 5490, 5488, 5478, 5768, 5592, 5600, 5580] dup_rows_per_dst=[2708, 2552, 2606, 2446, 2766, 2612, 2668, 2606] combine_rows_per_src=[5576, 5490, 5488, 5478, 5768, 5592, 5600, 5580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44566 payload_rows_remote=38962 dup_rows=20970 dup_rows_remote=18396 dedup_ratio=0.319977 router_tokens_per_expert=[510, 450, 510, 485, 510, 505, 505, 523, 538, 544, 543, 510, 523, 508, 490, 519, 490, 512, 517, 517, 491, 514, 502, 546, 499, 518, 505, 541, 522, 498, 522, 522, 484, 516, 551, 496, 491, 491, 533, 490, 501, 507, 524, 520, 505, 488, 516, 500, 494, 535, 512, 494, 564, 484, 541, 501, 507, 538, 524, 530, 537, 524, 506, 475] payload_rows_per_dst=[5492, 5632, 5604, 5568, 5470, 5574, 5636, 5590] dup_rows_per_dst=[2504, 2718, 2574, 2686, 2634, 2548, 2614, 2692] combine_rows_per_src=[5492, 5632, 5604, 5568, 5470, 5574, 5636, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=229256 combine_payload_bytes=279537664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279766920 ingress_bytes=499738624 link_bytes=499738624 token_bytes=559304584 weight_movement_bytes=220200960 total_nvlink_bytes=779505544 weight_to_token_ratio=0.393705 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.691276600e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=228552 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279020744 ingress_bytes=609093632 link_bytes=609093632 token_bytes=557812936 weight_movement_bytes=330301440 total_nvlink_bytes=888114376 weight_to_token_ratio=0.592137 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059314005e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38926 dup_rows=21088 dup_rows_remote=18566 dedup_ratio=0.321777 router_tokens_per_expert=[497, 515, 548, 566, 486, 491, 502, 521, 478, 515, 491, 519, 522, 498, 524, 552, 514, 536, 506, 534, 525, 466, 528, 536, 537, 529, 504, 505, 522, 539, 503, 485, 506, 501, 511, 536, 492, 459, 526, 491, 534, 511, 485, 500, 532, 539, 514, 510, 464, 505, 525, 497, 511, 491, 532, 536, 516, 512, 500, 512, 505, 485, 515, 521] payload_rows_per_dst=[5614, 5528, 5588, 5580, 5522, 5592, 5486, 5538] dup_rows_per_dst=[2638, 2670, 2702, 2668, 2522, 2658, 2636, 2594] combine_rows_per_src=[5614, 5528, 5588, 5580, 5522, 5592, 5486, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38934 dup_rows=21100 dup_rows_remote=18374 dedup_ratio=0.321960 router_tokens_per_expert=[521, 513, 478, 564, 511, 500, 509, 560, 512, 513, 538, 498, 488, 547, 530, 524, 554, 525, 525, 514, 506, 515, 515, 528, 512, 479, 531, 527, 492, 540, 521, 496, 498, 493, 485, 522, 508, 482, 483, 502, 532, 507, 543, 477, 519, 496, 473, 498, 508, 488, 498, 543, 513, 517, 536, 511, 545, 520, 486, 487, 501, 491, 508, 512] payload_rows_per_dst=[5644, 5636, 5608, 5540, 5432, 5570, 5502, 5504] dup_rows_per_dst=[2668, 2664, 2756, 2656, 2514, 2520, 2726, 2596] combine_rows_per_src=[5644, 5636, 5608, 5540, 5432, 5570, 5502, 5504] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280096768 dispatch_meta_bytes=229816 combine_payload_bytes=280096768 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280326584 ingress_bytes=500297728 link_bytes=500297728 token_bytes=560423352 weight_movement_bytes=220200960 total_nvlink_bytes=780624312 weight_to_token_ratio=0.392919 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701000338e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=229232 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279308144 ingress_bytes=389179392 link_bytes=389179392 token_bytes=558387056 weight_movement_bytes=110100480 total_nvlink_bytes=668487536 weight_to_token_ratio=0.197176 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768469717e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=228552 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279020744 ingress_bytes=609093632 link_bytes=609093632 token_bytes=557812936 weight_movement_bytes=330301440 total_nvlink_bytes=888114376 weight_to_token_ratio=0.592137 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059314005e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=229952 combine_payload_bytes=278462464 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278692416 ingress_bytes=498663424 link_bytes=498663424 token_bytes=557154880 weight_movement_bytes=220200960 total_nvlink_bytes=777355840 weight_to_token_ratio=0.395224 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672577104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280139776 dispatch_meta_bytes=229976 combine_payload_bytes=280139776 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280369752 ingress_bytes=390240256 link_bytes=390240256 token_bytes=560509528 weight_movement_bytes=110100480 total_nvlink_bytes=670610008 weight_to_token_ratio=0.196429 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786919887e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=229432 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279509048 ingress_bytes=499480576 link_bytes=499480576 token_bytes=558788664 weight_movement_bytes=220200960 total_nvlink_bytes=778989624 weight_to_token_ratio=0.394068 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686788721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279021568 dispatch_meta_bytes=229968 combine_payload_bytes=279021568 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279251536 ingress_bytes=499222528 link_bytes=499222528 token_bytes=558273104 weight_movement_bytes=220200960 total_nvlink_bytes=778474064 weight_to_token_ratio=0.394432 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682300842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44350 payload_rows_remote=38802 dup_rows=21186 dup_rows_remote=18478 dedup_ratio=0.323273 router_tokens_per_expert=[493, 531, 509, 527, 525, 519, 516, 524, 479, 526, 512, 515, 519, 535, 513, 504, 527, 493, 485, 521, 502, 530, 523, 518, 528, 492, 484, 507, 499, 511, 531, 481, 512, 518, 494, 503, 525, 525, 505, 546, 522, 511, 505, 526, 535, 502, 520, 495, 512, 519, 515, 483, 490, 535, 511, 508, 502, 521, 543, 493, 513, 508, 476, 516] payload_rows_per_dst=[5622, 5532, 5546, 5466, 5548, 5598, 5488, 5550] dup_rows_per_dst=[2666, 2674, 2652, 2600, 2708, 2634, 2658, 2594] combine_rows_per_src=[5622, 5532, 5546, 5466, 5548, 5598, 5488, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44566 payload_rows_remote=38962 dup_rows=20970 dup_rows_remote=18396 dedup_ratio=0.319977 router_tokens_per_expert=[510, 450, 510, 485, 510, 505, 505, 523, 538, 544, 543, 510, 523, 508, 490, 519, 490, 512, 517, 517, 491, 514, 502, 546, 499, 518, 505, 541, 522, 498, 522, 522, 484, 516, 551, 496, 491, 491, 533, 490, 501, 507, 524, 520, 505, 488, 516, 500, 494, 535, 512, 494, 564, 484, 541, 501, 507, 538, 524, 530, 537, 524, 506, 475] payload_rows_per_dst=[5492, 5632, 5604, 5568, 5470, 5574, 5636, 5590] dup_rows_per_dst=[2504, 2718, 2574, 2686, 2634, 2548, 2614, 2692] combine_rows_per_src=[5492, 5632, 5604, 5568, 5470, 5574, 5636, 5590] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=38880 dup_rows=21036 dup_rows_remote=18388 dedup_ratio=0.320984 router_tokens_per_expert=[486, 468, 533, 514, 522, 523, 490, 504, 495, 506, 544, 505, 512, 490, 489, 492, 549, 533, 476, 494, 522, 514, 511, 489, 538, 538, 529, 499, 485, 472, 520, 521, 494, 530, 537, 488, 500, 507, 494, 525, 512, 578, 507, 473, 531, 530, 496, 507, 512, 533, 533, 539, 487, 498, 493, 524, 481, 530, 529, 525, 533, 519, 541, 519] payload_rows_per_dst=[5526, 5474, 5552, 5542, 5504, 5620, 5580, 5702] dup_rows_per_dst=[2554, 2592, 2624, 2662, 2646, 2648, 2658, 2652] combine_rows_per_src=[5526, 5474, 5552, 5542, 5504, 5620, 5580, 5702] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5590381664 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2796338272 ingress_bytes=4775852032 link_bytes=4775852032 total_nvlink_bytes=7572190304 weight_to_token_ratio=0.354503 prefetch_share_of_ingress=0.207482 dedup_ratio=0.320425 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.305992176e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=229952 combine_payload_bytes=278462464 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278692416 ingress_bytes=498663424 link_bytes=498663424 token_bytes=557154880 weight_movement_bytes=220200960 total_nvlink_bytes=777355840 weight_to_token_ratio=0.395224 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672577104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5577965424 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=2790129520 ingress_bytes=4549443584 link_bytes=4549443584 total_nvlink_bytes=7339573104 weight_to_token_ratio=0.315815 prefetch_share_of_ingress=0.217808 dedup_ratio=0.321594 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.912230647e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38880 dup_rows=21122 dup_rows_remote=18540 dedup_ratio=0.322296 router_tokens_per_expert=[494, 507, 514, 539, 499, 523, 538, 536, 502, 537, 498, 472, 507, 518, 525, 504, 510, 487, 507, 523, 524, 514, 525, 494, 507, 533, 499, 510, 520, 485, 495, 504, 512, 506, 520, 565, 484, 511, 520, 500, 499, 480, 522, 525, 525, 508, 533, 529, 512, 534, 551, 502, 518, 485, 551, 468, 521, 462, 522, 513, 531, 492, 515, 502] payload_rows_per_dst=[5568, 5458, 5576, 5524, 5602, 5558, 5594, 5534] dup_rows_per_dst=[2654, 2668, 2574, 2582, 2712, 2684, 2666, 2582] combine_rows_per_src=[5568, 5458, 5576, 5524, 5602, 5558, 5594, 5534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38890 dup_rows=21052 dup_rows_remote=18562 dedup_ratio=0.321228 router_tokens_per_expert=[521, 543, 489, 544, 479, 493, 514, 501, 509, 525, 523, 521, 507, 529, 547, 553, 487, 504, 521, 497, 525, 526, 472, 499, 497, 517, 546, 502, 480, 481, 529, 514, 516, 505, 526, 531, 487, 554, 538, 467, 491, 505, 509, 470, 486, 521, 549, 531, 510, 503, 534, 518, 504, 511, 497, 465, 557, 483, 506, 487, 553, 508, 549, 502] payload_rows_per_dst=[5560, 5626, 5484, 5532, 5560, 5550, 5594, 5578] dup_rows_per_dst=[2608, 2802, 2578, 2600, 2688, 2574, 2490, 2712] combine_rows_per_src=[5560, 5626, 5484, 5532, 5560, 5550, 5594, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=229232 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279308144 ingress_bytes=389179392 link_bytes=389179392 token_bytes=558387056 weight_movement_bytes=110100480 total_nvlink_bytes=668487536 weight_to_token_ratio=0.197176 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768469717e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278132736 dispatch_meta_bytes=229120 combine_payload_bytes=278132736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278361856 ingress_bytes=388233216 link_bytes=388233216 token_bytes=556494592 weight_movement_bytes=110100480 total_nvlink_bytes=666595072 weight_to_token_ratio=0.197846 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752014160e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=229432 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279509048 ingress_bytes=499480576 link_bytes=499480576 token_bytes=558788664 weight_movement_bytes=220200960 total_nvlink_bytes=778989624 weight_to_token_ratio=0.394068 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686788721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229072 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278920912 ingress_bytes=388792320 link_bytes=388792320 token_bytes=557612752 weight_movement_bytes=110100480 total_nvlink_bytes=667713232 weight_to_token_ratio=0.197450 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761737898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38864 dup_rows=21170 dup_rows_remote=18562 dedup_ratio=0.323029 router_tokens_per_expert=[527, 500, 507, 512, 507, 512, 503, 472, 507, 489, 494, 540, 518, 518, 572, 525, 514, 533, 500, 514, 471, 531, 507, 523, 521, 535, 510, 510, 526, 512, 541, 511, 505, 485, 491, 530, 505, 461, 488, 548, 499, 505, 518, 526, 495, 513, 500, 514, 517, 551, 533, 525, 510, 493, 520, 488, 510, 515, 546, 495, 492, 507, 518, 503] payload_rows_per_dst=[5502, 5686, 5480, 5676, 5330, 5660, 5624, 5408] dup_rows_per_dst=[2608, 2730, 2624, 2948, 2404, 2648, 2620, 2588] combine_rows_per_src=[5502, 5686, 5480, 5676, 5330, 5660, 5624, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44572 payload_rows_remote=39082 dup_rows=20964 dup_rows_remote=18412 dedup_ratio=0.319885 router_tokens_per_expert=[537, 518, 539, 526, 499, 522, 499, 502, 492, 512, 493, 491, 524, 479, 529, 501, 521, 497, 525, 528, 531, 501, 494, 450, 498, 479, 483, 509, 485, 519, 505, 484, 529, 533, 556, 500, 533, 494, 585, 537, 499, 520, 515, 519, 504, 514, 524, 507, 511, 508, 531, 490, 521, 532, 528, 513, 520, 491, 524, 519, 518, 501, 528, 492] payload_rows_per_dst=[5576, 5490, 5488, 5478, 5768, 5592, 5600, 5580] dup_rows_per_dst=[2708, 2552, 2606, 2446, 2766, 2612, 2668, 2606] combine_rows_per_src=[5576, 5490, 5488, 5478, 5768, 5592, 5600, 5580] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=38912 dup_rows=20948 dup_rows_remote=18288 dedup_ratio=0.319641 router_tokens_per_expert=[485, 523, 456, 565, 531, 522, 532, 511, 488, 536, 516, 504, 505, 498, 489, 499, 517, 529, 510, 543, 543, 517, 520, 494, 523, 539, 511, 527, 507, 520, 534, 507, 499, 504, 511, 517, 503, 517, 487, 536, 501, 488, 510, 503, 522, 498, 504, 522, 522, 474, 495, 508, 498, 536, 504, 498, 504, 493, 519, 518, 488, 517, 549, 522] payload_rows_per_dst=[5554, 5532, 5624, 5676, 5546, 5514, 5530, 5612] dup_rows_per_dst=[2696, 2538, 2722, 2660, 2602, 2582, 2540, 2608] combine_rows_per_src=[5554, 5532, 5624, 5676, 5546, 5514, 5530, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38880 dup_rows=21122 dup_rows_remote=18540 dedup_ratio=0.322296 router_tokens_per_expert=[494, 507, 514, 539, 499, 523, 538, 536, 502, 537, 498, 472, 507, 518, 525, 504, 510, 487, 507, 523, 524, 514, 525, 494, 507, 533, 499, 510, 520, 485, 495, 504, 512, 506, 520, 565, 484, 511, 520, 500, 499, 480, 522, 525, 525, 508, 533, 529, 512, 534, 551, 502, 518, 485, 551, 468, 521, 462, 522, 513, 531, 492, 515, 502] payload_rows_per_dst=[5568, 5458, 5576, 5524, 5602, 5558, 5594, 5534] dup_rows_per_dst=[2654, 2668, 2574, 2582, 2712, 2684, 2666, 2582] combine_rows_per_src=[5568, 5458, 5576, 5524, 5602, 5558, 5594, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5589206392 prefetch_bytes=990904320 reduce_bytes=330301440 weight_movement_bytes=1321205760 egress_bytes=2795750776 ingress_bytes=4114661376 link_bytes=4114661376 total_nvlink_bytes=6910412152 weight_to_token_ratio=0.236385 prefetch_share_of_ingress=0.240823 dedup_ratio=0.320657 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=3 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.156072878e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38864 dup_rows=21170 dup_rows_remote=18562 dedup_ratio=0.323029 router_tokens_per_expert=[527, 500, 507, 512, 507, 512, 503, 472, 507, 489, 494, 540, 518, 518, 572, 525, 514, 533, 500, 514, 471, 531, 507, 523, 521, 535, 510, 510, 526, 512, 541, 511, 505, 485, 491, 530, 505, 461, 488, 548, 499, 505, 518, 526, 495, 513, 500, 514, 517, 551, 533, 525, 510, 493, 520, 488, 510, 515, 546, 495, 492, 507, 518, 503] payload_rows_per_dst=[5502, 5686, 5480, 5676, 5330, 5660, 5624, 5408] dup_rows_per_dst=[2608, 2730, 2624, 2948, 2404, 2648, 2620, 2588] combine_rows_per_src=[5502, 5686, 5480, 5676, 5330, 5660, 5624, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280139776 dispatch_meta_bytes=229976 combine_payload_bytes=280139776 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280369752 ingress_bytes=390240256 link_bytes=390240256 token_bytes=560509528 weight_movement_bytes=110100480 total_nvlink_bytes=670610008 weight_to_token_ratio=0.196429 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786919887e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38748 dup_rows=21170 dup_rows_remote=18474 dedup_ratio=0.323029 router_tokens_per_expert=[520, 462, 482, 502, 546, 505, 537, 531, 533, 489, 495, 490, 498, 515, 541, 534, 517, 511, 506, 510, 535, 522, 533, 523, 496, 550, 500, 481, 497, 499, 496, 462, 512, 506, 531, 546, 522, 520, 542, 503, 514, 487, 520, 471, 512, 507, 532, 479, 526, 517, 515, 466, 506, 539, 524, 532, 533, 483, 479, 538, 518, 518, 530, 522] payload_rows_per_dst=[5610, 5532, 5618, 5424, 5608, 5376, 5554, 5644] dup_rows_per_dst=[2560, 2658, 2696, 2538, 2756, 2668, 2696, 2598] combine_rows_per_src=[5610, 5532, 5618, 5424, 5608, 5376, 5554, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=228800 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279150016 ingress_bytes=389021696 link_bytes=389021696 token_bytes=558071232 weight_movement_bytes=110100480 total_nvlink_bytes=668171712 weight_to_token_ratio=0.197288 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765727124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38926 dup_rows=21088 dup_rows_remote=18566 dedup_ratio=0.321777 router_tokens_per_expert=[497, 515, 548, 566, 486, 491, 502, 521, 478, 515, 491, 519, 522, 498, 524, 552, 514, 536, 506, 534, 525, 466, 528, 536, 537, 529, 504, 505, 522, 539, 503, 485, 506, 501, 511, 536, 492, 459, 526, 491, 534, 511, 485, 500, 532, 539, 514, 510, 464, 505, 525, 497, 511, 491, 532, 536, 516, 512, 500, 512, 505, 485, 515, 521] payload_rows_per_dst=[5614, 5528, 5588, 5580, 5522, 5592, 5486, 5538] dup_rows_per_dst=[2638, 2670, 2702, 2668, 2522, 2658, 2636, 2594] combine_rows_per_src=[5614, 5528, 5588, 5580, 5522, 5592, 5486, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38890 dup_rows=21052 dup_rows_remote=18562 dedup_ratio=0.321228 router_tokens_per_expert=[521, 543, 489, 544, 479, 493, 514, 501, 509, 525, 523, 521, 507, 529, 547, 553, 487, 504, 521, 497, 525, 526, 472, 499, 497, 517, 546, 502, 480, 481, 529, 514, 516, 505, 526, 531, 487, 554, 538, 467, 491, 505, 509, 470, 486, 521, 549, 531, 510, 503, 534, 518, 504, 511, 497, 465, 557, 483, 506, 487, 553, 508, 549, 502] payload_rows_per_dst=[5560, 5626, 5484, 5532, 5560, 5550, 5594, 5578] dup_rows_per_dst=[2608, 2802, 2578, 2600, 2688, 2574, 2490, 2712] combine_rows_per_src=[5560, 5626, 5484, 5532, 5560, 5550, 5594, 5578] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229680 combine_payload_bytes=278691840 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278921520 ingress_bytes=498892800 link_bytes=498892800 token_bytes=557613360 weight_movement_bytes=220200960 total_nvlink_bytes=777814320 weight_to_token_ratio=0.394899 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676566330e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229808 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278993328 ingress_bytes=498964480 link_bytes=498964480 token_bytes=557756848 weight_movement_bytes=220200960 total_nvlink_bytes=777957808 weight_to_token_ratio=0.394797 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677812963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5589206392 prefetch_bytes=990904320 reduce_bytes=330301440 weight_movement_bytes=1321205760 egress_bytes=2795750776 ingress_bytes=4114661376 link_bytes=4114661376 total_nvlink_bytes=6910412152 weight_to_token_ratio=0.236385 prefetch_share_of_ingress=0.240823 dedup_ratio=0.320657 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=3 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.156072878e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229704 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278806856 ingress_bytes=498778112 link_bytes=498778112 token_bytes=557384008 weight_movement_bytes=220200960 total_nvlink_bytes=777584968 weight_to_token_ratio=0.395061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.674571717e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=228800 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279150016 ingress_bytes=389021696 link_bytes=389021696 token_bytes=558071232 weight_movement_bytes=110100480 total_nvlink_bytes=668171712 weight_to_token_ratio=0.197288 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765727124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38748 dup_rows=21170 dup_rows_remote=18474 dedup_ratio=0.323029 router_tokens_per_expert=[520, 462, 482, 502, 546, 505, 537, 531, 533, 489, 495, 490, 498, 515, 541, 534, 517, 511, 506, 510, 535, 522, 533, 523, 496, 550, 500, 481, 497, 499, 496, 462, 512, 506, 531, 546, 522, 520, 542, 503, 514, 487, 520, 471, 512, 507, 532, 479, 526, 517, 515, 466, 506, 539, 524, 532, 533, 483, 479, 538, 518, 518, 530, 522] payload_rows_per_dst=[5610, 5532, 5618, 5424, 5608, 5376, 5554, 5644] dup_rows_per_dst=[2560, 2658, 2696, 2538, 2756, 2668, 2696, 2598] combine_rows_per_src=[5610, 5532, 5618, 5424, 5608, 5376, 5554, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5590381664 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2796338272 ingress_bytes=4775852032 link_bytes=4775852032 total_nvlink_bytes=7572190304 weight_to_token_ratio=0.354503 prefetch_share_of_ingress=0.207482 dedup_ratio=0.320425 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.305992176e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5580771440 prefetch_bytes=770703360 reduce_bytes=2202009600 weight_movement_bytes=2972712960 egress_bytes=2791530608 ingress_bytes=5761953792 link_bytes=5761953792 total_nvlink_bytes=8553484400 weight_to_token_ratio=0.532671 prefetch_share_of_ingress=0.133757 dedup_ratio=0.320358 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=20 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.002098532e-04 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279021568 dispatch_meta_bytes=229968 combine_payload_bytes=279021568 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279251536 ingress_bytes=499222528 link_bytes=499222528 token_bytes=558273104 weight_movement_bytes=220200960 total_nvlink_bytes=778474064 weight_to_token_ratio=0.394432 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682300842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229808 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278993328 ingress_bytes=498964480 link_bytes=498964480 token_bytes=557756848 weight_movement_bytes=220200960 total_nvlink_bytes=777957808 weight_to_token_ratio=0.394797 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677812963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5569759480 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=2786023672 ingress_bytes=4545343488 link_bytes=4545343488 total_nvlink_bytes=7331367160 weight_to_token_ratio=0.316281 prefetch_share_of_ingress=0.193782 dedup_ratio=0.321741 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.905099906e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5580745056 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=2791518560 ingress_bytes=4881135616 link_bytes=4881135616 total_nvlink_bytes=7672654176 weight_to_token_ratio=0.374844 prefetch_share_of_ingress=0.203007 dedup_ratio=0.320990 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.489097645e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277745664 dispatch_meta_bytes=228888 combine_payload_bytes=277745664 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=277974552 ingress_bytes=608047104 link_bytes=608047104 token_bytes=555720216 weight_movement_bytes=330301440 total_nvlink_bytes=886021656 weight_to_token_ratio=0.594366 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057493920e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5577965424 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=2790129520 ingress_bytes=4549443584 link_bytes=4549443584 total_nvlink_bytes=7339573104 weight_to_token_ratio=0.315815 prefetch_share_of_ingress=0.217808 dedup_ratio=0.321594 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.912230647e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588116432 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=2795205584 ingress_bytes=4664619008 link_bytes=4664619008 total_nvlink_bytes=7459824592 weight_to_token_ratio=0.334944 prefetch_share_of_ingress=0.212430 dedup_ratio=0.320560 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.112539653e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5580745056 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=2791518560 ingress_bytes=4881135616 link_bytes=4881135616 total_nvlink_bytes=7672654176 weight_to_token_ratio=0.374844 prefetch_share_of_ingress=0.203007 dedup_ratio=0.320990 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.489097645e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5580771440 prefetch_bytes=770703360 reduce_bytes=2202009600 weight_movement_bytes=2972712960 egress_bytes=2791530608 ingress_bytes=5761953792 link_bytes=5761953792 total_nvlink_bytes=8553484400 weight_to_token_ratio=0.532671 prefetch_share_of_ingress=0.133757 dedup_ratio=0.320358 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=20 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.002098532e-04 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5587915144 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=2795104648 ingress_bytes=4444317696 link_bytes=4444317696 total_nvlink_bytes=7239422344 weight_to_token_ratio=0.295550 prefetch_share_of_ingress=0.247733 dedup_ratio=0.320499 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.729399438e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none - [2026-08-26 16:58:53] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 364.3 | throughput per GPU (TFLOP/s/GPU): 339.0 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.033287E+01 | loss scale: 1.0 | grad norm: 8.146 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44778 payload_rows_remote=39230 dup_rows=20758 dup_rows_remote=18224 dedup_ratio=0.316742 router_tokens_per_expert=[539, 539, 469, 500, 506, 499, 527, 497, 477, 509, 491, 492, 525, 559, 521, 503, 507, 486, 546, 516, 555, 476, 462, 509, 525, 521, 521, 516, 509, 474, 491, 484, 466, 513, 511, 486, 517, 510, 507, 543, 497, 525, 479, 533, 533, 547, 524, 526, 493, 483, 531, 534, 541, 489, 559, 522, 519, 531, 493, 498, 538, 523, 526, 520] payload_rows_per_dst=[5558, 5618, 5582, 5548, 5586, 5632, 5606, 5648] dup_rows_per_dst=[2594, 2536, 2532, 2534, 2520, 2696, 2698, 2648] combine_rows_per_src=[5558, 5618, 5582, 5548, 5586, 5632, 5606, 5648] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44498 payload_rows_remote=38918 dup_rows=21038 dup_rows_remote=18432 dedup_ratio=0.321014 router_tokens_per_expert=[499, 497, 505, 502, 501, 529, 499, 528, 504, 484, 498, 529, 491, 501, 495, 541, 483, 511, 561, 529, 491, 478, 488, 552, 531, 507, 509, 522, 510, 563, 508, 480, 519, 508, 480, 547, 484, 472, 464, 550, 533, 489, 520, 547, 479, 503, 521, 529, 566, 527, 478, 490, 547, 529, 525, 479, 516, 554, 522, 486, 517, 508, 530, 523] payload_rows_per_dst=[5472, 5574, 5580, 5570, 5500, 5622, 5612, 5568] dup_rows_per_dst=[2648, 2512, 2606, 2690, 2548, 2620, 2670, 2744] combine_rows_per_src=[5472, 5574, 5580, 5570, 5500, 5622, 5612, 5568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38856 dup_rows=21058 dup_rows_remote=18294 dedup_ratio=0.321320 router_tokens_per_expert=[511, 524, 468, 512, 488, 495, 514, 447, 522, 504, 508, 507, 542, 530, 549, 531, 531, 519, 524, 540, 520, 535, 483, 486, 506, 532, 514, 510, 495, 540, 522, 537, 527, 487, 526, 496, 515, 483, 487, 521, 475, 508, 488, 491, 520, 521, 536, 507, 527, 509, 531, 506, 495, 496, 517, 486, 535, 511, 507, 565, 508, 498, 509, 534] payload_rows_per_dst=[5486, 5622, 5594, 5606, 5520, 5526, 5516, 5608] dup_rows_per_dst=[2432, 2764, 2682, 2706, 2564, 2566, 2618, 2726] combine_rows_per_src=[5486, 5622, 5594, 5606, 5520, 5526, 5516, 5608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44516 payload_rows_remote=38844 dup_rows=21020 dup_rows_remote=18162 dedup_ratio=0.320740 router_tokens_per_expert=[570, 542, 497, 498, 547, 493, 532, 530, 552, 488, 507, 511, 508, 486, 506, 532, 541, 477, 514, 550, 520, 572, 500, 499, 543, 492, 496, 477, 509, 503, 537, 549, 480, 532, 505, 498, 499, 487, 481, 518, 506, 513, 505, 498, 477, 540, 491, 493, 543, 521, 482, 528, 492, 520, 480, 494, 530, 543, 523, 561, 508, 457, 499, 486] payload_rows_per_dst=[5672, 5566, 5632, 5678, 5482, 5530, 5394, 5562] dup_rows_per_dst=[2858, 2654, 2568, 2826, 2518, 2516, 2434, 2646] combine_rows_per_src=[5672, 5566, 5632, 5678, 5482, 5530, 5394, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=39094 dup_rows=20938 dup_rows_remote=18430 dedup_ratio=0.319489 router_tokens_per_expert=[492, 523, 515, 518, 541, 495, 533, 532, 502, 545, 539, 513, 515, 514, 507, 493, 513, 535, 520, 484, 501, 492, 525, 530, 485, 509, 536, 520, 477, 564, 516, 492, 468, 542, 513, 495, 529, 504, 524, 504, 527, 507, 492, 519, 494, 485, 496, 486, 542, 518, 528, 497, 483, 553, 530, 526, 518, 495, 497, 503, 519, 493, 482, 523] payload_rows_per_dst=[5606, 5580, 5556, 5638, 5588, 5504, 5670, 5456] dup_rows_per_dst=[2692, 2676, 2644, 2560, 2570, 2508, 2684, 2604] combine_rows_per_src=[5606, 5580, 5556, 5638, 5588, 5504, 5670, 5456] + [2026-08-26 18:10:58] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 363.3 | throughput per GPU (TFLOP/s/GPU): 339.9 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.032770E+01 | loss scale: 1.0 | grad norm: 8.154 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=38940 dup_rows=20946 dup_rows_remote=18350 dedup_ratio=0.319611 router_tokens_per_expert=[514, 518, 508, 538, 519, 505, 505, 492, 518, 538, 515, 505, 506, 484, 518, 526, 515, 540, 489, 504, 497, 549, 506, 538, 490, 513, 507, 498, 488, 489, 542, 480, 543, 561, 528, 507, 511, 510, 515, 514, 503, 513, 518, 502, 512, 503, 488, 520, 496, 521, 502, 549, 497, 532, 512, 514, 532, 470, 509, 492, 501, 531, 487, 521] payload_rows_per_dst=[5574, 5512, 5608, 5508, 5630, 5574, 5650, 5534] dup_rows_per_dst=[2624, 2708, 2668, 2506, 2748, 2544, 2596, 2552] combine_rows_per_src=[5574, 5512, 5608, 5508, 5630, 5574, 5650, 5534] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44400 payload_rows_remote=38780 dup_rows=21136 dup_rows_remote=18334 dedup_ratio=0.322510 router_tokens_per_expert=[519, 527, 488, 532, 500, 490, 512, 507, 487, 511, 526, 491, 533, 522, 530, 504, 556, 545, 501, 547, 549, 497, 510, 524, 505, 487, 491, 480, 511, 522, 510, 512, 526, 541, 526, 491, 520, 528, 541, 538, 509, 493, 509, 496, 528, 530, 512, 520, 511, 484, 490, 497, 522, 539, 489, 500, 489, 486, 520, 482, 507, 498, 509, 511] payload_rows_per_dst=[5554, 5492, 5698, 5486, 5620, 5594, 5434, 5522] dup_rows_per_dst=[2596, 2716, 2760, 2550, 2802, 2600, 2630, 2482] combine_rows_per_src=[5554, 5492, 5698, 5486, 5620, 5594, 5434, 5522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44690 payload_rows_remote=39102 dup_rows=20846 dup_rows_remote=18198 dedup_ratio=0.318085 router_tokens_per_expert=[505, 491, 548, 510, 489, 518, 519, 505, 515, 504, 499, 528, 505, 506, 506, 496, 492, 536, 506, 541, 493, 502, 499, 513, 544, 480, 506, 527, 483, 530, 483, 512, 500, 528, 527, 512, 524, 497, 510, 516, 510, 481, 547, 497, 514, 517, 504, 523, 529, 517, 497, 486, 576, 508, 526, 513, 480, 531, 563, 480, 538, 484, 531, 511] payload_rows_per_dst=[5552, 5534, 5406, 5518, 5660, 5784, 5648, 5588] dup_rows_per_dst=[2618, 2472, 2420, 2612, 2680, 2720, 2676, 2648] combine_rows_per_src=[5552, 5534, 5406, 5518, 5660, 5784, 5648, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281200640 dispatch_meta_bytes=229816 combine_payload_bytes=281200640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281430456 ingress_bytes=391301120 link_bytes=391301120 token_bytes=562631096 weight_movement_bytes=110100480 total_nvlink_bytes=672731576 weight_to_token_ratio=0.195689 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.805370057e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44778 payload_rows_remote=39230 dup_rows=20758 dup_rows_remote=18224 dedup_ratio=0.316742 router_tokens_per_expert=[539, 539, 469, 500, 506, 499, 527, 497, 477, 509, 491, 492, 525, 559, 521, 503, 507, 486, 546, 516, 555, 476, 462, 509, 525, 521, 521, 516, 509, 474, 491, 484, 466, 513, 511, 486, 517, 510, 507, 543, 497, 525, 479, 533, 533, 547, 524, 526, 493, 483, 531, 534, 541, 489, 559, 522, 519, 531, 493, 498, 538, 523, 526, 520] payload_rows_per_dst=[5558, 5618, 5582, 5548, 5586, 5632, 5606, 5648] dup_rows_per_dst=[2594, 2536, 2532, 2534, 2520, 2696, 2698, 2648] combine_rows_per_src=[5558, 5618, 5582, 5548, 5586, 5632, 5606, 5648] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=229400 combine_payload_bytes=278964224 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=279193624 ingress_bytes=829466624 link_bytes=829466624 token_bytes=558157848 weight_movement_bytes=550502400 total_nvlink_bytes=1108660248 weight_to_token_ratio=0.986284 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.442578883e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=228024 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278661816 ingress_bytes=388534272 link_bytes=388534272 token_bytes=557095608 weight_movement_bytes=110100480 total_nvlink_bytes=667196088 weight_to_token_ratio=0.197633 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757250019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278519808 dispatch_meta_bytes=228600 combine_payload_bytes=278519808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278748408 ingress_bytes=498720768 link_bytes=498720768 token_bytes=557268216 weight_movement_bytes=220200960 total_nvlink_bytes=777469176 weight_to_token_ratio=0.395144 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673574410e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=230096 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280455888 ingress_bytes=390326272 link_bytes=390326272 token_bytes=560681680 weight_movement_bytes=110100480 total_nvlink_bytes=670782160 weight_to_token_ratio=0.196369 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788415846e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38856 dup_rows=21058 dup_rows_remote=18294 dedup_ratio=0.321320 router_tokens_per_expert=[511, 524, 468, 512, 488, 495, 514, 447, 522, 504, 508, 507, 542, 530, 549, 531, 531, 519, 524, 540, 520, 535, 483, 486, 506, 532, 514, 510, 495, 540, 522, 537, 527, 487, 526, 496, 515, 483, 487, 521, 475, 508, 488, 491, 520, 521, 536, 507, 527, 509, 531, 506, 495, 496, 517, 486, 535, 511, 507, 565, 508, 498, 509, 534] payload_rows_per_dst=[5486, 5622, 5594, 5606, 5520, 5526, 5516, 5608] dup_rows_per_dst=[2432, 2764, 2682, 2706, 2564, 2566, 2618, 2726] combine_rows_per_src=[5486, 5622, 5594, 5606, 5520, 5526, 5516, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=39094 dup_rows=20938 dup_rows_remote=18430 dedup_ratio=0.319489 router_tokens_per_expert=[492, 523, 515, 518, 541, 495, 533, 532, 502, 545, 539, 513, 515, 514, 507, 493, 513, 535, 520, 484, 501, 492, 525, 530, 485, 509, 536, 520, 477, 564, 516, 492, 468, 542, 513, 495, 529, 504, 524, 504, 527, 507, 492, 519, 494, 485, 496, 486, 542, 518, 528, 497, 483, 553, 530, 526, 518, 495, 497, 503, 519, 493, 482, 523] payload_rows_per_dst=[5606, 5580, 5556, 5638, 5588, 5504, 5670, 5456] dup_rows_per_dst=[2692, 2676, 2644, 2560, 2570, 2508, 2684, 2604] combine_rows_per_src=[5606, 5580, 5556, 5638, 5588, 5504, 5670, 5456] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=229160 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279351080 ingress_bytes=499322880 link_bytes=499322880 token_bytes=558473000 weight_movement_bytes=220200960 total_nvlink_bytes=778673960 weight_to_token_ratio=0.394291 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684046128e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44516 payload_rows_remote=38844 dup_rows=21020 dup_rows_remote=18162 dedup_ratio=0.320740 router_tokens_per_expert=[570, 542, 497, 498, 547, 493, 532, 530, 552, 488, 507, 511, 508, 486, 506, 532, 541, 477, 514, 550, 520, 572, 500, 499, 543, 492, 496, 477, 509, 503, 537, 549, 480, 532, 505, 498, 499, 487, 481, 518, 506, 513, 505, 498, 477, 540, 491, 493, 543, 521, 482, 528, 492, 520, 480, 494, 530, 543, 523, 561, 508, 457, 499, 486] payload_rows_per_dst=[5672, 5566, 5632, 5678, 5482, 5530, 5394, 5562] dup_rows_per_dst=[2858, 2654, 2568, 2826, 2518, 2516, 2434, 2646] combine_rows_per_src=[5672, 5566, 5632, 5678, 5482, 5530, 5394, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=228456 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278203496 ingress_bytes=388075520 link_bytes=388075520 token_bytes=556178536 weight_movement_bytes=110100480 total_nvlink_bytes=666279016 weight_to_token_ratio=0.197959 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749271567e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=229200 combine_payload_bytes=280283136 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=280512336 ingress_bytes=390383616 link_bytes=390383616 token_bytes=560795472 weight_movement_bytes=110100480 total_nvlink_bytes=670895952 weight_to_token_ratio=0.196329 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789413153e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38858 dup_rows=21166 dup_rows_remote=18512 dedup_ratio=0.322968 router_tokens_per_expert=[521, 523, 546, 488, 482, 506, 521, 478, 545, 488, 538, 522, 493, 530, 533, 541, 504, 470, 483, 526, 583, 509, 475, 519, 510, 478, 491, 503, 500, 546, 537, 518, 524, 525, 534, 462, 516, 493, 504, 528, 517, 497, 542, 529, 528, 526, 545, 523, 474, 503, 508, 480, 519, 512, 519, 507, 525, 488, 480, 504, 504, 524, 508, 513] payload_rows_per_dst=[5518, 5614, 5542, 5512, 5494, 5640, 5496, 5554] dup_rows_per_dst=[2612, 2766, 2596, 2654, 2678, 2774, 2548, 2538] combine_rows_per_src=[5518, 5614, 5542, 5512, 5494, 5640, 5496, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44282 payload_rows_remote=38604 dup_rows=21254 dup_rows_remote=18494 dedup_ratio=0.324310 router_tokens_per_expert=[490, 521, 532, 499, 515, 521, 481, 522, 554, 513, 502, 511, 504, 522, 537, 552, 520, 499, 555, 528, 503, 540, 505, 569, 508, 520, 538, 501, 543, 521, 496, 523, 464, 505, 513, 478, 501, 515, 481, 514, 509, 508, 496, 547, 503, 481, 497, 512, 508, 527, 485, 539, 499, 549, 487, 502, 535, 518, 476, 510, 490, 482, 498, 494] payload_rows_per_dst=[5464, 5648, 5678, 5542, 5454, 5446, 5598, 5452] dup_rows_per_dst=[2698, 2742, 2760, 2758, 2488, 2660, 2594, 2554] combine_rows_per_src=[5464, 5648, 5678, 5542, 5454, 5446, 5598, 5452] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38984 dup_rows=20988 dup_rows_remote=18326 dedup_ratio=0.320251 router_tokens_per_expert=[494, 532, 507, 484, 514, 482, 514, 526, 509, 465, 543, 534, 538, 505, 529, 490, 479, 504, 534, 498, 506, 517, 502, 510, 508, 539, 526, 512, 494, 493, 540, 521, 535, 475, 473, 494, 527, 544, 528, 528, 512, 541, 482, 503, 539, 535, 507, 494, 520, 507, 517, 498, 544, 521, 553, 507, 483, 466, 487, 520, 539, 515, 508, 517] payload_rows_per_dst=[5596, 5564, 5518, 5608, 5596, 5566, 5642, 5458] dup_rows_per_dst=[2510, 2662, 2582, 2658, 2612, 2660, 2692, 2612] combine_rows_per_src=[5596, 5564, 5518, 5608, 5596, 5566, 5642, 5458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44356 payload_rows_remote=38676 dup_rows=21180 dup_rows_remote=18372 dedup_ratio=0.323181 router_tokens_per_expert=[514, 522, 531, 530, 514, 505, 512, 546, 487, 484, 513, 489, 560, 494, 529, 480, 505, 499, 493, 534, 523, 530, 524, 533, 523, 510, 530, 530, 522, 514, 483, 498, 535, 514, 517, 484, 512, 523, 480, 458, 495, 512, 536, 478, 530, 521, 518, 506, 498, 518, 499, 519, 555, 484, 493, 516, 514, 519, 529, 508, 496, 506, 519, 515] payload_rows_per_dst=[5680, 5562, 5516, 5532, 5696, 5562, 5250, 5558] dup_rows_per_dst=[2808, 2592, 2544, 2688, 3006, 2630, 2258, 2654] combine_rows_per_src=[5680, 5562, 5516, 5532, 5696, 5562, 5250, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38826 dup_rows=21144 dup_rows_remote=18520 dedup_ratio=0.322632 router_tokens_per_expert=[493, 527, 510, 511, 523, 499, 523, 495, 517, 471, 545, 542, 507, 497, 522, 519, 523, 494, 512, 511, 502, 529, 486, 546, 541, 497, 488, 550, 509, 495, 518, 535, 551, 517, 519, 481, 518, 518, 520, 471, 495, 495, 481, 490, 527, 514, 517, 513, 512, 467, 545, 501, 492, 573, 523, 502, 481, 502, 544, 524, 528, 457, 552, 501] payload_rows_per_dst=[5578, 5620, 5538, 5564, 5566, 5514, 5502, 5510] dup_rows_per_dst=[2584, 2620, 2668, 2702, 2624, 2550, 2728, 2668] combine_rows_per_src=[5578, 5620, 5538, 5564, 5566, 5514, 5502, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281200640 dispatch_meta_bytes=229816 combine_payload_bytes=281200640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281430456 ingress_bytes=391301120 link_bytes=391301120 token_bytes=562631096 weight_movement_bytes=110100480 total_nvlink_bytes=672731576 weight_to_token_ratio=0.195689 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.805370057e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278519808 dispatch_meta_bytes=228600 combine_payload_bytes=278519808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278748408 ingress_bytes=498720768 link_bytes=498720768 token_bytes=557268216 weight_movement_bytes=220200960 total_nvlink_bytes=777469176 weight_to_token_ratio=0.395144 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673574410e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=230096 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280455888 ingress_bytes=390326272 link_bytes=390326272 token_bytes=560681680 weight_movement_bytes=110100480 total_nvlink_bytes=670782160 weight_to_token_ratio=0.196369 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788415846e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=228024 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278661816 ingress_bytes=388534272 link_bytes=388534272 token_bytes=557095608 weight_movement_bytes=110100480 total_nvlink_bytes=667196088 weight_to_token_ratio=0.197633 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757250019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44498 payload_rows_remote=38918 dup_rows=21038 dup_rows_remote=18432 dedup_ratio=0.321014 router_tokens_per_expert=[499, 497, 505, 502, 501, 529, 499, 528, 504, 484, 498, 529, 491, 501, 495, 541, 483, 511, 561, 529, 491, 478, 488, 552, 531, 507, 509, 522, 510, 563, 508, 480, 519, 508, 480, 547, 484, 472, 464, 550, 533, 489, 520, 547, 479, 503, 521, 529, 566, 527, 478, 490, 547, 529, 525, 479, 516, 554, 522, 486, 517, 508, 530, 523] payload_rows_per_dst=[5472, 5574, 5580, 5570, 5500, 5622, 5612, 5568] dup_rows_per_dst=[2648, 2512, 2606, 2690, 2548, 2620, 2670, 2744] combine_rows_per_src=[5472, 5574, 5580, 5570, 5500, 5622, 5612, 5568] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38994 dup_rows=20926 dup_rows_remote=18454 dedup_ratio=0.319305 router_tokens_per_expert=[523, 493, 534, 532, 513, 495, 506, 520, 484, 545, 511, 502, 546, 483, 469, 491, 535, 533, 494, 490, 553, 525, 506, 535, 494, 516, 488, 544, 527, 547, 479, 479, 482, 530, 505, 534, 493, 541, 515, 511, 525, 518, 516, 503, 542, 527, 507, 514, 502, 503, 495, 486, 507, 484, 529, 538, 524, 527, 466, 518, 523, 496, 522, 493] payload_rows_per_dst=[5568, 5522, 5556, 5576, 5638, 5626, 5616, 5508] dup_rows_per_dst=[2664, 2540, 2786, 2572, 2584, 2678, 2472, 2630] combine_rows_per_src=[5568, 5522, 5556, 5576, 5638, 5626, 5616, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44730 payload_rows_remote=38962 dup_rows=20806 dup_rows_remote=17816 dedup_ratio=0.317474 router_tokens_per_expert=[485, 550, 491, 527, 479, 504, 500, 530, 494, 507, 490, 505, 524, 504, 503, 525, 493, 534, 519, 491, 525, 513, 511, 552, 496, 538, 510, 494, 543, 550, 510, 489, 518, 485, 515, 479, 493, 493, 508, 499, 473, 521, 531, 469, 499, 488, 514, 504, 528, 529, 510, 562, 535, 550, 493, 520, 536, 530, 551, 481, 490, 518, 537, 523] payload_rows_per_dst=[5598, 5434, 5636, 5442, 5522, 5556, 5774, 5768] dup_rows_per_dst=[2534, 2244, 2636, 2484, 2458, 2542, 2918, 2990] combine_rows_per_src=[5598, 5434, 5636, 5442, 5522, 5556, 5774, 5768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38826 dup_rows=21144 dup_rows_remote=18520 dedup_ratio=0.322632 router_tokens_per_expert=[493, 527, 510, 511, 523, 499, 523, 495, 517, 471, 545, 542, 507, 497, 522, 519, 523, 494, 512, 511, 502, 529, 486, 546, 541, 497, 488, 550, 509, 495, 518, 535, 551, 517, 519, 481, 518, 518, 520, 471, 495, 495, 481, 490, 527, 514, 517, 513, 512, 467, 545, 501, 492, 573, 523, 502, 481, 502, 544, 524, 528, 457, 552, 501] payload_rows_per_dst=[5578, 5620, 5538, 5564, 5566, 5514, 5502, 5510] dup_rows_per_dst=[2584, 2620, 2668, 2702, 2624, 2550, 2728, 2668] combine_rows_per_src=[5578, 5620, 5538, 5564, 5566, 5514, 5502, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=229792 combine_payload_bytes=279508992 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279738784 ingress_bytes=499709952 link_bytes=499709952 token_bytes=559247776 weight_movement_bytes=220200960 total_nvlink_bytes=779448736 weight_to_token_ratio=0.393745 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.690777947e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38984 dup_rows=20988 dup_rows_remote=18326 dedup_ratio=0.320251 router_tokens_per_expert=[494, 532, 507, 484, 514, 482, 514, 526, 509, 465, 543, 534, 538, 505, 529, 490, 479, 504, 534, 498, 506, 517, 502, 510, 508, 539, 526, 512, 494, 493, 540, 521, 535, 475, 473, 494, 527, 544, 528, 528, 512, 541, 482, 503, 539, 535, 507, 494, 520, 507, 517, 498, 544, 521, 553, 507, 483, 466, 487, 520, 539, 515, 508, 517] payload_rows_per_dst=[5596, 5564, 5518, 5608, 5596, 5566, 5642, 5458] dup_rows_per_dst=[2510, 2662, 2582, 2658, 2612, 2660, 2692, 2612] combine_rows_per_src=[5596, 5564, 5518, 5608, 5596, 5566, 5642, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38858 dup_rows=21166 dup_rows_remote=18512 dedup_ratio=0.322968 router_tokens_per_expert=[521, 523, 546, 488, 482, 506, 521, 478, 545, 488, 538, 522, 493, 530, 533, 541, 504, 470, 483, 526, 583, 509, 475, 519, 510, 478, 491, 503, 500, 546, 537, 518, 524, 525, 534, 462, 516, 493, 504, 528, 517, 497, 542, 529, 528, 526, 545, 523, 474, 503, 508, 480, 519, 512, 519, 507, 525, 488, 480, 504, 504, 524, 508, 513] payload_rows_per_dst=[5518, 5614, 5542, 5512, 5494, 5640, 5496, 5554] dup_rows_per_dst=[2612, 2766, 2596, 2654, 2678, 2774, 2548, 2538] combine_rows_per_src=[5518, 5614, 5542, 5512, 5494, 5640, 5496, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=229400 combine_payload_bytes=278964224 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=279193624 ingress_bytes=829466624 link_bytes=829466624 token_bytes=558157848 weight_movement_bytes=550502400 total_nvlink_bytes=1108660248 weight_to_token_ratio=0.986284 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.442578883e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38920 dup_rows=21054 dup_rows_remote=18484 dedup_ratio=0.321259 router_tokens_per_expert=[478, 488, 486, 533, 500, 533, 544, 500, 534, 558, 515, 518, 545, 514, 525, 494, 454, 491, 483, 556, 485, 492, 518, 511, 507, 571, 544, 507, 499, 507, 493, 494, 538, 479, 484, 510, 498, 522, 513, 516, 506, 476, 537, 523, 493, 498, 514, 519, 502, 522, 506, 533, 548, 536, 530, 547, 508, 481, 509, 525, 496, 510, 506, 506] payload_rows_per_dst=[5530, 5686, 5472, 5574, 5544, 5562, 5628, 5486] dup_rows_per_dst=[2594, 2720, 2508, 2670, 2576, 2570, 2820, 2596] combine_rows_per_src=[5530, 5686, 5472, 5574, 5544, 5562, 5628, 5486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44356 payload_rows_remote=38676 dup_rows=21180 dup_rows_remote=18372 dedup_ratio=0.323181 router_tokens_per_expert=[514, 522, 531, 530, 514, 505, 512, 546, 487, 484, 513, 489, 560, 494, 529, 480, 505, 499, 493, 534, 523, 530, 524, 533, 523, 510, 530, 530, 522, 514, 483, 498, 535, 514, 517, 484, 512, 523, 480, 458, 495, 512, 536, 478, 530, 521, 518, 506, 498, 518, 499, 519, 555, 484, 493, 516, 514, 519, 529, 508, 496, 506, 519, 515] payload_rows_per_dst=[5680, 5562, 5516, 5532, 5696, 5562, 5250, 5558] dup_rows_per_dst=[2808, 2592, 2544, 2688, 3006, 2630, 2258, 2654] combine_rows_per_src=[5680, 5562, 5516, 5532, 5696, 5562, 5250, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38842 dup_rows=21116 dup_rows_remote=18438 dedup_ratio=0.322205 router_tokens_per_expert=[514, 493, 513, 528, 491, 514, 457, 506, 516, 514, 498, 525, 536, 464, 541, 511, 507, 515, 502, 491, 514, 537, 556, 525, 542, 513, 543, 496, 456, 513, 505, 517, 490, 503, 465, 492, 494, 528, 509, 510, 520, 471, 522, 506, 519, 527, 512, 529, 486, 541, 507, 506, 512, 545, 554, 477, 504, 518, 521, 528, 497, 540, 506, 576] payload_rows_per_dst=[5478, 5548, 5560, 5540, 5446, 5592, 5578, 5678] dup_rows_per_dst=[2554, 2662, 2734, 2630, 2536, 2620, 2678, 2702] combine_rows_per_src=[5478, 5548, 5560, 5540, 5446, 5592, 5578, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229384 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278534152 ingress_bytes=388405248 link_bytes=388405248 token_bytes=556838920 weight_movement_bytes=110100480 total_nvlink_bytes=666939400 weight_to_token_ratio=0.197724 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755006080e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=227112 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279506728 ingress_bytes=389380096 link_bytes=389380096 token_bytes=558786344 weight_movement_bytes=110100480 total_nvlink_bytes=668886824 weight_to_token_ratio=0.197035 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771960290e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=229480 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278763624 ingress_bytes=608835584 link_bytes=608835584 token_bytes=557297768 weight_movement_bytes=330301440 total_nvlink_bytes=887599208 weight_to_token_ratio=0.592684 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058865217e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44730 payload_rows_remote=38962 dup_rows=20806 dup_rows_remote=17816 dedup_ratio=0.317474 router_tokens_per_expert=[485, 550, 491, 527, 479, 504, 500, 530, 494, 507, 490, 505, 524, 504, 503, 525, 493, 534, 519, 491, 525, 513, 511, 552, 496, 538, 510, 494, 543, 550, 510, 489, 518, 485, 515, 479, 493, 493, 508, 499, 473, 521, 531, 469, 499, 488, 514, 504, 528, 529, 510, 562, 535, 550, 493, 520, 536, 530, 551, 481, 490, 518, 537, 523] payload_rows_per_dst=[5598, 5434, 5636, 5442, 5522, 5556, 5774, 5768] dup_rows_per_dst=[2534, 2244, 2636, 2484, 2458, 2542, 2918, 2990] combine_rows_per_src=[5598, 5434, 5636, 5442, 5522, 5556, 5774, 5768] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276713472 dispatch_meta_bytes=228392 combine_payload_bytes=276713472 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=276941864 ingress_bytes=607014912 link_bytes=607014912 token_bytes=553655336 weight_movement_bytes=330301440 total_nvlink_bytes=883956776 weight_to_token_ratio=0.596583 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055698769e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=229240 combine_payload_bytes=279437312 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279666552 ingress_bytes=499638272 link_bytes=499638272 token_bytes=559103864 weight_movement_bytes=220200960 total_nvlink_bytes=779304824 weight_to_token_ratio=0.393846 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689531314e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=229480 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278763624 ingress_bytes=608835584 link_bytes=608835584 token_bytes=557297768 weight_movement_bytes=330301440 total_nvlink_bytes=887599208 weight_to_token_ratio=0.592684 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058865217e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=229120 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278648576 ingress_bytes=498620416 link_bytes=498620416 token_bytes=557068032 weight_movement_bytes=220200960 total_nvlink_bytes=777268992 weight_to_token_ratio=0.395286 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671829124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277229568 dispatch_meta_bytes=228192 combine_payload_bytes=277229568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277457760 ingress_bytes=387330048 link_bytes=387330048 token_bytes=554687328 weight_movement_bytes=110100480 total_nvlink_bytes=664787808 weight_to_token_ratio=0.198491 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.736306583e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229384 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278534152 ingress_bytes=388405248 link_bytes=388405248 token_bytes=556838920 weight_movement_bytes=110100480 total_nvlink_bytes=666939400 weight_to_token_ratio=0.197724 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755006080e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=229792 combine_payload_bytes=279508992 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279738784 ingress_bytes=499709952 link_bytes=499709952 token_bytes=559247776 weight_movement_bytes=220200960 total_nvlink_bytes=779448736 weight_to_token_ratio=0.393745 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.690777947e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278978560 dispatch_meta_bytes=229616 combine_payload_bytes=278978560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279208176 ingress_bytes=389079040 link_bytes=389079040 token_bytes=558186736 weight_movement_bytes=110100480 total_nvlink_bytes=668287216 weight_to_token_ratio=0.197247 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766724431e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=227112 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279506728 ingress_bytes=389380096 link_bytes=389380096 token_bytes=558786344 weight_movement_bytes=110100480 total_nvlink_bytes=668886824 weight_to_token_ratio=0.197035 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771960290e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38644 dup_rows=21142 dup_rows_remote=18304 dedup_ratio=0.322601 router_tokens_per_expert=[533, 536, 488, 507, 537, 500, 506, 539, 485, 470, 511, 474, 510, 509, 551, 529, 493, 479, 489, 497, 496, 460, 563, 482, 553, 531, 581, 540, 523, 527, 517, 522, 497, 504, 522, 491, 545, 506, 479, 511, 508, 541, 519, 485, 540, 495, 500, 484, 547, 547, 494, 503, 523, 528, 497, 487, 521, 552, 504, 511, 517, 481, 480, 511] payload_rows_per_dst=[5648, 5514, 5364, 5750, 5570, 5494, 5554, 5500] dup_rows_per_dst=[2644, 2564, 2554, 2838, 2540, 2650, 2698, 2654] combine_rows_per_src=[5648, 5514, 5364, 5750, 5570, 5494, 5554, 5500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38992 dup_rows=21026 dup_rows_remote=18310 dedup_ratio=0.320831 router_tokens_per_expert=[468, 524, 551, 516, 492, 513, 538, 532, 531, 509, 521, 524, 517, 541, 525, 534, 495, 516, 534, 495, 471, 498, 521, 472, 545, 497, 520, 495, 507, 481, 517, 545, 513, 495, 513, 507, 506, 571, 519, 493, 484, 519, 515, 516, 534, 500, 506, 555, 513, 505, 521, 490, 553, 481, 496, 523, 488, 523, 460, 530, 478, 503, 511, 502] payload_rows_per_dst=[5600, 5644, 5516, 5564, 5518, 5608, 5552, 5508] dup_rows_per_dst=[2668, 2760, 2488, 2650, 2716, 2650, 2612, 2482] combine_rows_per_src=[5600, 5644, 5516, 5564, 5518, 5608, 5552, 5508] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38824 dup_rows=21116 dup_rows_remote=18516 dedup_ratio=0.322205 router_tokens_per_expert=[522, 521, 506, 555, 504, 493, 506, 476, 503, 518, 469, 493, 486, 516, 475, 520, 511, 530, 505, 523, 508, 502, 510, 509, 478, 477, 527, 501, 502, 513, 513, 467, 495, 503, 498, 509, 525, 499, 512, 529, 522, 536, 511, 537, 518, 535, 499, 540, 528, 526, 499, 559, 530, 501, 477, 595, 524, 491, 521, 541, 510, 495, 524, 540] payload_rows_per_dst=[5556, 5464, 5596, 5472, 5494, 5614, 5628, 5596] dup_rows_per_dst=[2610, 2496, 2600, 2484, 2646, 2782, 2802, 2696] combine_rows_per_src=[5556, 5464, 5596, 5472, 5494, 5614, 5628, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44644 payload_rows_remote=39094 dup_rows=20892 dup_rows_remote=18306 dedup_ratio=0.318787 router_tokens_per_expert=[529, 523, 498, 498, 526, 503, 524, 500, 516, 490, 549, 564, 561, 515, 481, 515, 506, 506, 521, 512, 513, 537, 516, 525, 501, 499, 483, 540, 470, 510, 496, 516, 543, 524, 485, 549, 490, 523, 483, 516, 499, 526, 530, 525, 539, 505, 511, 479, 477, 480, 548, 504, 508, 516, 510, 501, 507, 515, 511, 493, 509, 498, 509, 512] payload_rows_per_dst=[5582, 5736, 5562, 5502, 5582, 5632, 5498, 5550] dup_rows_per_dst=[2534, 2730, 2710, 2502, 2644, 2596, 2590, 2586] combine_rows_per_src=[5582, 5736, 5562, 5502, 5582, 5632, 5498, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44282 payload_rows_remote=38604 dup_rows=21254 dup_rows_remote=18494 dedup_ratio=0.324310 router_tokens_per_expert=[490, 521, 532, 499, 515, 521, 481, 522, 554, 513, 502, 511, 504, 522, 537, 552, 520, 499, 555, 528, 503, 540, 505, 569, 508, 520, 538, 501, 543, 521, 496, 523, 464, 505, 513, 478, 501, 515, 481, 514, 509, 508, 496, 547, 503, 481, 497, 512, 508, 527, 485, 539, 499, 549, 487, 502, 535, 518, 476, 510, 490, 482, 498, 494] payload_rows_per_dst=[5464, 5648, 5678, 5542, 5454, 5446, 5598, 5452] dup_rows_per_dst=[2698, 2742, 2760, 2758, 2488, 2660, 2594, 2554] combine_rows_per_src=[5464, 5648, 5678, 5542, 5454, 5446, 5598, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44282 payload_rows_remote=38724 dup_rows=21254 dup_rows_remote=18664 dedup_ratio=0.324310 router_tokens_per_expert=[521, 532, 527, 518, 554, 529, 536, 477, 530, 501, 499, 515, 504, 512, 521, 492, 546, 537, 509, 508, 512, 474, 502, 528, 483, 530, 509, 520, 524, 509, 518, 513, 529, 512, 503, 521, 498, 498, 498, 529, 511, 496, 556, 501, 496, 536, 478, 513, 559, 499, 517, 512, 495, 479, 507, 527, 502, 475, 496, 469, 515, 514, 551, 486] payload_rows_per_dst=[5586, 5558, 5534, 5562, 5502, 5528, 5598, 5414] dup_rows_per_dst=[2802, 2590, 2698, 2650, 2674, 2646, 2592, 2602] combine_rows_per_src=[5586, 5558, 5534, 5562, 5502, 5528, 5598, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38644 dup_rows=21142 dup_rows_remote=18304 dedup_ratio=0.322601 router_tokens_per_expert=[533, 536, 488, 507, 537, 500, 506, 539, 485, 470, 511, 474, 510, 509, 551, 529, 493, 479, 489, 497, 496, 460, 563, 482, 553, 531, 581, 540, 523, 527, 517, 522, 497, 504, 522, 491, 545, 506, 479, 511, 508, 541, 519, 485, 540, 495, 500, 484, 547, 547, 494, 503, 523, 528, 497, 487, 521, 552, 504, 511, 517, 481, 480, 511] payload_rows_per_dst=[5648, 5514, 5364, 5750, 5570, 5494, 5554, 5500] dup_rows_per_dst=[2644, 2564, 2554, 2838, 2540, 2650, 2698, 2654] combine_rows_per_src=[5648, 5514, 5364, 5750, 5570, 5494, 5554, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=39112 dup_rows=20936 dup_rows_remote=18450 dedup_ratio=0.319458 router_tokens_per_expert=[452, 515, 548, 534, 484, 528, 521, 500, 526, 526, 519, 507, 546, 515, 478, 490, 526, 544, 496, 509, 511, 520, 503, 516, 479, 510, 553, 472, 537, 517, 480, 504, 528, 506, 539, 500, 532, 488, 471, 506, 522, 509, 477, 532, 525, 535, 481, 541, 487, 493, 503, 484, 475, 532, 502, 511, 523, 498, 508, 544, 553, 538, 539, 520] payload_rows_per_dst=[5578, 5570, 5610, 5584, 5550, 5542, 5488, 5678] dup_rows_per_dst=[2586, 2644, 2640, 2520, 2590, 2702, 2486, 2768] combine_rows_per_src=[5578, 5570, 5610, 5584, 5550, 5542, 5488, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279494656 dispatch_meta_bytes=229208 combine_payload_bytes=279494656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279723864 ingress_bytes=389595136 link_bytes=389595136 token_bytes=559218520 weight_movement_bytes=110100480 total_nvlink_bytes=669319000 weight_to_token_ratio=0.196883 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775700189e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38960 dup_rows=21054 dup_rows_remote=18562 dedup_ratio=0.321259 router_tokens_per_expert=[457, 526, 521, 486, 503, 483, 511, 520, 522, 475, 510, 501, 485, 500, 537, 491, 492, 515, 494, 487, 500, 508, 481, 551, 557, 510, 500, 511, 535, 530, 546, 513, 541, 536, 489, 526, 486, 524, 507, 484, 512, 536, 500, 518, 547, 501, 534, 500, 511, 459, 502, 514, 499, 517, 539, 529, 515, 536, 538, 545, 488, 546, 495, 536] payload_rows_per_dst=[5522, 5514, 5636, 5674, 5658, 5476, 5438, 5564] dup_rows_per_dst=[2492, 2528, 2810, 2730, 2896, 2510, 2460, 2628] combine_rows_per_src=[5522, 5514, 5636, 5674, 5658, 5476, 5438, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38842 dup_rows=21116 dup_rows_remote=18438 dedup_ratio=0.322205 router_tokens_per_expert=[514, 493, 513, 528, 491, 514, 457, 506, 516, 514, 498, 525, 536, 464, 541, 511, 507, 515, 502, 491, 514, 537, 556, 525, 542, 513, 543, 496, 456, 513, 505, 517, 490, 503, 465, 492, 494, 528, 509, 510, 520, 471, 522, 506, 519, 527, 512, 529, 486, 541, 507, 506, 512, 545, 554, 477, 504, 518, 521, 528, 497, 540, 506, 576] payload_rows_per_dst=[5478, 5548, 5560, 5540, 5446, 5592, 5578, 5678] dup_rows_per_dst=[2554, 2662, 2734, 2630, 2536, 2620, 2678, 2702] combine_rows_per_src=[5478, 5548, 5560, 5540, 5446, 5592, 5578, 5678] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277000192 dispatch_meta_bytes=227792 combine_payload_bytes=277000192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277227984 ingress_bytes=497201152 link_bytes=497201152 token_bytes=554228176 weight_movement_bytes=220200960 total_nvlink_bytes=774429136 weight_to_token_ratio=0.397311 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.647145789e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44554 payload_rows_remote=38960 dup_rows=20982 dup_rows_remote=18438 dedup_ratio=0.320160 router_tokens_per_expert=[533, 511, 515, 490, 510, 528, 508, 535, 523, 526, 524, 506, 512, 514, 519, 490, 551, 498, 487, 491, 482, 525, 513, 540, 491, 503, 491, 503, 501, 505, 482, 518, 500, 538, 516, 526, 485, 489, 491, 512, 484, 514, 516, 506, 515, 516, 516, 502, 530, 528, 495, 526, 530, 484, 504, 532, 497, 519, 510, 519, 547, 510, 538, 548] payload_rows_per_dst=[5570, 5556, 5544, 5530, 5542, 5594, 5542, 5676] dup_rows_per_dst=[2690, 2672, 2630, 2458, 2572, 2544, 2716, 2700] combine_rows_per_src=[5570, 5556, 5544, 5530, 5542, 5594, 5542, 5676] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44644 payload_rows_remote=39094 dup_rows=20892 dup_rows_remote=18306 dedup_ratio=0.318787 router_tokens_per_expert=[529, 523, 498, 498, 526, 503, 524, 500, 516, 490, 549, 564, 561, 515, 481, 515, 506, 506, 521, 512, 513, 537, 516, 525, 501, 499, 483, 540, 470, 510, 496, 516, 543, 524, 485, 549, 490, 523, 483, 516, 499, 526, 530, 525, 539, 505, 511, 479, 477, 480, 548, 504, 508, 516, 510, 501, 507, 515, 511, 493, 509, 498, 509, 512] payload_rows_per_dst=[5582, 5736, 5562, 5502, 5582, 5632, 5498, 5550] dup_rows_per_dst=[2534, 2730, 2710, 2502, 2644, 2596, 2590, 2586] combine_rows_per_src=[5582, 5736, 5562, 5502, 5582, 5632, 5498, 5550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279494656 dispatch_meta_bytes=229208 combine_payload_bytes=279494656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279723864 ingress_bytes=389595136 link_bytes=389595136 token_bytes=559218520 weight_movement_bytes=110100480 total_nvlink_bytes=669319000 weight_to_token_ratio=0.196883 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775700189e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=229600 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280455392 ingress_bytes=610527232 link_bytes=610527232 token_bytes=560681184 weight_movement_bytes=330301440 total_nvlink_bytes=890982624 weight_to_token_ratio=0.589107 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061807271e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276713472 dispatch_meta_bytes=228392 combine_payload_bytes=276713472 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=276941864 ingress_bytes=607014912 link_bytes=607014912 token_bytes=553655336 weight_movement_bytes=330301440 total_nvlink_bytes=883956776 weight_to_token_ratio=0.596583 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055698769e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=229360 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278519792 ingress_bytes=388390912 link_bytes=388390912 token_bytes=556810224 weight_movement_bytes=110100480 total_nvlink_bytes=666910704 weight_to_token_ratio=0.197734 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754756753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=229552 combine_payload_bytes=277573632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277803184 ingress_bytes=387674112 link_bytes=387674112 token_bytes=555376816 weight_movement_bytes=110100480 total_nvlink_bytes=665477296 weight_to_token_ratio=0.198245 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.742290422e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=230088 combine_payload_bytes=279265280 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279495368 ingress_bytes=499466240 link_bytes=499466240 token_bytes=558760648 weight_movement_bytes=220200960 total_nvlink_bytes=778961608 weight_to_token_ratio=0.394088 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686539394e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=229120 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278648576 ingress_bytes=498620416 link_bytes=498620416 token_bytes=557068032 weight_movement_bytes=220200960 total_nvlink_bytes=777268992 weight_to_token_ratio=0.395286 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671829124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277000192 dispatch_meta_bytes=227792 combine_payload_bytes=277000192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277227984 ingress_bytes=497201152 link_bytes=497201152 token_bytes=554228176 weight_movement_bytes=220200960 total_nvlink_bytes=774429136 weight_to_token_ratio=0.397311 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.647145789e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280354816 dispatch_meta_bytes=230248 combine_payload_bytes=280354816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280585064 ingress_bytes=390455296 link_bytes=390455296 token_bytes=560939880 weight_movement_bytes=110100480 total_nvlink_bytes=671040360 weight_to_token_ratio=0.196279 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790659786e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=229592 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279494872 ingress_bytes=499466240 link_bytes=499466240 token_bytes=558760152 weight_movement_bytes=220200960 total_nvlink_bytes=778961112 weight_to_token_ratio=0.394089 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686539394e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=229600 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280455392 ingress_bytes=610527232 link_bytes=610527232 token_bytes=560681184 weight_movement_bytes=330301440 total_nvlink_bytes=890982624 weight_to_token_ratio=0.589107 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061807271e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38990 dup_rows=20926 dup_rows_remote=18214 dedup_ratio=0.319305 router_tokens_per_expert=[504, 553, 495, 551, 518, 522, 499, 525, 518, 463, 498, 502, 518, 508, 524, 501, 501, 525, 491, 530, 558, 517, 516, 465, 502, 538, 473, 494, 547, 541, 552, 519, 506, 493, 498, 561, 508, 509, 520, 529, 495, 501, 505, 534, 523, 492, 517, 528, 511, 514, 481, 500, 515, 515, 534, 521, 512, 491, 499, 468, 491, 493, 520, 516] payload_rows_per_dst=[5628, 5522, 5564, 5620, 5628, 5590, 5576, 5482] dup_rows_per_dst=[2706, 2542, 2642, 2712, 2620, 2600, 2606, 2498] combine_rows_per_src=[5628, 5522, 5564, 5620, 5628, 5590, 5576, 5482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=230088 combine_payload_bytes=279265280 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279495368 ingress_bytes=499466240 link_bytes=499466240 token_bytes=558760648 weight_movement_bytes=220200960 total_nvlink_bytes=778961608 weight_to_token_ratio=0.394088 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686539394e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38900 dup_rows=21002 dup_rows_remote=18220 dedup_ratio=0.320465 router_tokens_per_expert=[526, 529, 494, 531, 502, 496, 501, 480, 535, 500, 530, 501, 489, 523, 477, 498, 498, 499, 536, 541, 501, 473, 525, 494, 511, 547, 479, 524, 521, 515, 479, 481, 524, 532, 551, 508, 517, 525, 498, 553, 491, 542, 520, 517, 490, 514, 530, 504, 556, 506, 516, 487, 503, 486, 512, 510, 524, 489, 544, 494, 525, 510, 521, 533] payload_rows_per_dst=[5552, 5494, 5548, 5536, 5634, 5552, 5576, 5642] dup_rows_per_dst=[2566, 2612, 2586, 2578, 2782, 2664, 2576, 2638] combine_rows_per_src=[5552, 5494, 5548, 5536, 5634, 5552, 5576, 5642] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38880 dup_rows=21072 dup_rows_remote=18384 dedup_ratio=0.321533 router_tokens_per_expert=[525, 500, 530, 528, 530, 518, 505, 525, 532, 519, 518, 502, 531, 485, 546, 503, 502, 528, 521, 505, 485, 484, 511, 495, 491, 496, 500, 499, 502, 505, 526, 526, 504, 507, 543, 451, 546, 481, 504, 524, 526, 515, 534, 504, 514, 503, 474, 509, 525, 482, 470, 520, 508, 552, 534, 531, 526, 515, 532, 502, 503, 490, 524, 542] payload_rows_per_dst=[5676, 5584, 5492, 5494, 5564, 5500, 5498, 5656] dup_rows_per_dst=[2646, 2688, 2570, 2596, 2556, 2658, 2746, 2612] combine_rows_per_src=[5676, 5584, 5492, 5494, 5564, 5500, 5498, 5656] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=228816 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279709136 ingress_bytes=389580800 link_bytes=389580800 token_bytes=559189456 weight_movement_bytes=110100480 total_nvlink_bytes=669289936 weight_to_token_ratio=0.196893 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775450862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44310 payload_rows_remote=38772 dup_rows=21226 dup_rows_remote=18458 dedup_ratio=0.323883 router_tokens_per_expert=[497, 530, 495, 521, 512, 532, 525, 541, 493, 510, 498, 479, 495, 504, 493, 483, 552, 506, 541, 489, 505, 473, 480, 560, 454, 540, 502, 498, 516, 522, 534, 515, 520, 515, 502, 525, 517, 489, 517, 494, 527, 483, 512, 496, 525, 530, 531, 529, 521, 516, 516, 515, 546, 521, 485, 491, 523, 507, 511, 563, 511, 507, 490, 538] payload_rows_per_dst=[5538, 5744, 5568, 5572, 5488, 5266, 5584, 5550] dup_rows_per_dst=[2768, 3026, 2644, 2590, 2670, 2140, 2638, 2750] combine_rows_per_src=[5538, 5744, 5568, 5572, 5488, 5266, 5584, 5550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=38934 dup_rows=21024 dup_rows_remote=18286 dedup_ratio=0.320801 router_tokens_per_expert=[531, 510, 559, 538, 487, 509, 508, 514, 504, 490, 499, 500, 506, 512, 490, 505, 502, 512, 537, 528, 499, 520, 524, 536, 503, 497, 505, 491, 475, 517, 496, 477, 521, 515, 505, 532, 525, 526, 517, 553, 539, 529, 481, 521, 523, 492, 488, 511, 521, 495, 540, 514, 539, 530, 486, 524, 511, 514, 540, 482, 522, 515, 483, 493] payload_rows_per_dst=[5632, 5518, 5578, 5426, 5660, 5538, 5660, 5500] dup_rows_per_dst=[2680, 2494, 2738, 2496, 2728, 2630, 2638, 2620] combine_rows_per_src=[5632, 5518, 5578, 5426, 5660, 5538, 5660, 5500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=39112 dup_rows=20936 dup_rows_remote=18450 dedup_ratio=0.319458 router_tokens_per_expert=[452, 515, 548, 534, 484, 528, 521, 500, 526, 526, 519, 507, 546, 515, 478, 490, 526, 544, 496, 509, 511, 520, 503, 516, 479, 510, 553, 472, 537, 517, 480, 504, 528, 506, 539, 500, 532, 488, 471, 506, 522, 509, 477, 532, 525, 535, 481, 541, 487, 493, 503, 484, 475, 532, 502, 511, 523, 498, 508, 544, 553, 538, 539, 520] payload_rows_per_dst=[5578, 5570, 5610, 5584, 5550, 5542, 5488, 5678] dup_rows_per_dst=[2586, 2644, 2640, 2520, 2590, 2702, 2486, 2768] combine_rows_per_src=[5578, 5570, 5610, 5584, 5550, 5542, 5488, 5678] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38904 dup_rows=21090 dup_rows_remote=18552 dedup_ratio=0.321808 router_tokens_per_expert=[493, 492, 500, 511, 536, 487, 479, 480, 510, 511, 476, 522, 514, 494, 512, 475, 529, 493, 563, 499, 511, 497, 508, 519, 537, 529, 522, 498, 528, 534, 558, 545, 487, 509, 485, 514, 520, 534, 536, 524, 497, 541, 493, 512, 483, 546, 537, 546, 552, 507, 540, 481, 522, 487, 525, 494, 510, 530, 504, 508, 496, 513, 486, 487] payload_rows_per_dst=[5328, 5524, 5554, 5724, 5378, 5862, 5534, 5542] dup_rows_per_dst=[2418, 2504, 2598, 2876, 2506, 2930, 2720, 2538] combine_rows_per_src=[5328, 5524, 5554, 5724, 5378, 5862, 5534, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38990 dup_rows=20926 dup_rows_remote=18214 dedup_ratio=0.319305 router_tokens_per_expert=[504, 553, 495, 551, 518, 522, 499, 525, 518, 463, 498, 502, 518, 508, 524, 501, 501, 525, 491, 530, 558, 517, 516, 465, 502, 538, 473, 494, 547, 541, 552, 519, 506, 493, 498, 561, 508, 509, 520, 529, 495, 501, 505, 534, 523, 492, 517, 528, 511, 514, 481, 500, 515, 515, 534, 521, 512, 491, 499, 468, 491, 493, 520, 516] payload_rows_per_dst=[5628, 5522, 5564, 5620, 5628, 5590, 5576, 5482] dup_rows_per_dst=[2706, 2542, 2642, 2712, 2620, 2600, 2606, 2498] combine_rows_per_src=[5628, 5522, 5564, 5620, 5628, 5590, 5576, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38824 dup_rows=21116 dup_rows_remote=18516 dedup_ratio=0.322205 router_tokens_per_expert=[522, 521, 506, 555, 504, 493, 506, 476, 503, 518, 469, 493, 486, 516, 475, 520, 511, 530, 505, 523, 508, 502, 510, 509, 478, 477, 527, 501, 502, 513, 513, 467, 495, 503, 498, 509, 525, 499, 512, 529, 522, 536, 511, 537, 518, 535, 499, 540, 528, 526, 499, 559, 530, 501, 477, 595, 524, 491, 521, 541, 510, 495, 524, 540] payload_rows_per_dst=[5556, 5464, 5596, 5472, 5494, 5614, 5628, 5596] dup_rows_per_dst=[2610, 2496, 2600, 2484, 2646, 2782, 2802, 2696] combine_rows_per_src=[5556, 5464, 5596, 5472, 5494, 5614, 5628, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38880 dup_rows=21072 dup_rows_remote=18384 dedup_ratio=0.321533 router_tokens_per_expert=[525, 500, 530, 528, 530, 518, 505, 525, 532, 519, 518, 502, 531, 485, 546, 503, 502, 528, 521, 505, 485, 484, 511, 495, 491, 496, 500, 499, 502, 505, 526, 526, 504, 507, 543, 451, 546, 481, 504, 524, 526, 515, 534, 504, 514, 503, 474, 509, 525, 482, 470, 520, 508, 552, 534, 531, 526, 515, 532, 502, 503, 490, 524, 542] payload_rows_per_dst=[5676, 5584, 5492, 5494, 5564, 5500, 5498, 5656] dup_rows_per_dst=[2646, 2688, 2570, 2596, 2556, 2658, 2746, 2612] combine_rows_per_src=[5676, 5584, 5492, 5494, 5564, 5500, 5498, 5656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=228480 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279063680 ingress_bytes=499036160 link_bytes=499036160 token_bytes=557898880 weight_movement_bytes=220200960 total_nvlink_bytes=778099840 weight_to_token_ratio=0.394697 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679059596e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=39070 dup_rows=20918 dup_rows_remote=18318 dedup_ratio=0.319183 router_tokens_per_expert=[469, 471, 534, 503, 499, 516, 539, 512, 540, 491, 517, 537, 519, 525, 504, 528, 512, 509, 515, 505, 530, 499, 505, 509, 538, 522, 527, 498, 488, 498, 546, 492, 507, 507, 517, 534, 503, 503, 495, 510, 527, 539, 494, 523, 533, 473, 545, 520, 489, 493, 499, 510, 502, 532, 485, 564, 520, 488, 530, 492, 509, 502, 564, 462] payload_rows_per_dst=[5532, 5704, 5612, 5570, 5542, 5568, 5548, 5542] dup_rows_per_dst=[2554, 2618, 2556, 2648, 2610, 2740, 2600, 2592] combine_rows_per_src=[5532, 5704, 5612, 5570, 5542, 5568, 5548, 5542] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38780 dup_rows=21130 dup_rows_remote=18394 dedup_ratio=0.322418 router_tokens_per_expert=[553, 511, 466, 545, 491, 510, 553, 488, 529, 492, 511, 493, 505, 475, 536, 494, 493, 521, 502, 526, 543, 491, 489, 537, 481, 506, 512, 526, 520, 521, 536, 504, 527, 563, 481, 512, 505, 469, 508, 526, 542, 495, 548, 535, 490, 529, 519, 523, 520, 522, 502, 477, 524, 515, 519, 526, 536, 462, 530, 517, 498, 498, 489, 501] payload_rows_per_dst=[5552, 5512, 5530, 5592, 5552, 5626, 5572, 5470] dup_rows_per_dst=[2682, 2558, 2674, 2620, 2630, 2736, 2638, 2592] combine_rows_per_src=[5552, 5512, 5530, 5592, 5552, 5626, 5572, 5470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44310 payload_rows_remote=38772 dup_rows=21226 dup_rows_remote=18458 dedup_ratio=0.323883 router_tokens_per_expert=[497, 530, 495, 521, 512, 532, 525, 541, 493, 510, 498, 479, 495, 504, 493, 483, 552, 506, 541, 489, 505, 473, 480, 560, 454, 540, 502, 498, 516, 522, 534, 515, 520, 515, 502, 525, 517, 489, 517, 494, 527, 483, 512, 496, 525, 530, 531, 529, 521, 516, 516, 515, 546, 521, 485, 491, 523, 507, 511, 563, 511, 507, 490, 538] payload_rows_per_dst=[5538, 5744, 5568, 5572, 5488, 5266, 5584, 5550] dup_rows_per_dst=[2768, 3026, 2644, 2590, 2670, 2140, 2638, 2750] combine_rows_per_src=[5538, 5744, 5568, 5572, 5488, 5266, 5584, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229824 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279093696 ingress_bytes=388964352 link_bytes=388964352 token_bytes=557957568 weight_movement_bytes=110100480 total_nvlink_bytes=668058048 weight_to_token_ratio=0.197328 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764729818e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=228480 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279063680 ingress_bytes=499036160 link_bytes=499036160 token_bytes=557898880 weight_movement_bytes=220200960 total_nvlink_bytes=778099840 weight_to_token_ratio=0.394697 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679059596e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=228920 combine_payload_bytes=277917696 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278146616 ingress_bytes=718319616 link_bytes=718319616 token_bytes=556064312 weight_movement_bytes=440401920 total_nvlink_bytes=996466232 weight_to_token_ratio=0.791998 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249275955e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=228816 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279709136 ingress_bytes=389580800 link_bytes=389580800 token_bytes=559189456 weight_movement_bytes=110100480 total_nvlink_bytes=669289936 weight_to_token_ratio=0.196893 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775450862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278290432 dispatch_meta_bytes=229360 combine_payload_bytes=278290432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278519792 ingress_bytes=388390912 link_bytes=388390912 token_bytes=556810224 weight_movement_bytes=110100480 total_nvlink_bytes=666910704 weight_to_token_ratio=0.197734 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754756753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229056 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278920896 ingress_bytes=388792320 link_bytes=388792320 token_bytes=557612736 weight_movement_bytes=110100480 total_nvlink_bytes=667713216 weight_to_token_ratio=0.197450 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761737898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=228880 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279307792 ingress_bytes=499279872 link_bytes=499279872 token_bytes=558386704 weight_movement_bytes=220200960 total_nvlink_bytes=778587664 weight_to_token_ratio=0.394352 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683298148e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280354816 dispatch_meta_bytes=230248 combine_payload_bytes=280354816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280585064 ingress_bytes=390455296 link_bytes=390455296 token_bytes=560939880 weight_movement_bytes=110100480 total_nvlink_bytes=671040360 weight_to_token_ratio=0.196279 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790659786e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229824 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279093696 ingress_bytes=388964352 link_bytes=388964352 token_bytes=557957568 weight_movement_bytes=110100480 total_nvlink_bytes=668058048 weight_to_token_ratio=0.197328 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764729818e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280053760 dispatch_meta_bytes=229552 combine_payload_bytes=280053760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280283312 ingress_bytes=390154240 link_bytes=390154240 token_bytes=560337072 weight_movement_bytes=110100480 total_nvlink_bytes=670437552 weight_to_token_ratio=0.196490 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785423927e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=228696 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278203736 ingress_bytes=498176000 link_bytes=498176000 token_bytes=556178776 weight_movement_bytes=220200960 total_nvlink_bytes=776379736 weight_to_token_ratio=0.395918 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.664099999e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38888 dup_rows=21084 dup_rows_remote=18412 dedup_ratio=0.321716 router_tokens_per_expert=[525, 517, 503, 530, 524, 517, 480, 503, 469, 522, 487, 491, 511, 479, 562, 528, 543, 476, 502, 485, 508, 502, 492, 536, 489, 533, 546, 507, 520, 515, 522, 486, 493, 531, 496, 507, 522, 540, 475, 516, 508, 497, 516, 464, 546, 528, 516, 536, 487, 518, 518, 521, 511, 526, 513, 481, 488, 518, 540, 584, 500, 493, 543, 526] payload_rows_per_dst=[5606, 5490, 5502, 5564, 5600, 5526, 5552, 5612] dup_rows_per_dst=[2592, 2608, 2586, 2672, 2560, 2696, 2598, 2772] combine_rows_per_src=[5606, 5490, 5502, 5564, 5600, 5526, 5552, 5612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=228920 combine_payload_bytes=277917696 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278146616 ingress_bytes=718319616 link_bytes=718319616 token_bytes=556064312 weight_movement_bytes=440401920 total_nvlink_bytes=996466232 weight_to_token_ratio=0.791998 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249275955e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=39032 dup_rows=21072 dup_rows_remote=18430 dedup_ratio=0.321533 router_tokens_per_expert=[508, 502, 481, 538, 522, 501, 523, 521, 487, 534, 514, 484, 525, 539, 540, 478, 545, 524, 544, 508, 494, 526, 527, 546, 532, 526, 502, 486, 537, 499, 532, 476, 512, 511, 509, 506, 518, 493, 488, 500, 509, 529, 531, 492, 492, 539, 513, 473, 498, 511, 502, 539, 507, 537, 514, 525, 537, 478, 516, 491, 486, 495, 506, 510] payload_rows_per_dst=[5596, 5594, 5718, 5538, 5432, 5558, 5558, 5470] dup_rows_per_dst=[2596, 2608, 2710, 2642, 2642, 2598, 2708, 2568] combine_rows_per_src=[5596, 5594, 5718, 5538, 5432, 5558, 5558, 5470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229200 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278978384 ingress_bytes=388849664 link_bytes=388849664 token_bytes=557727568 weight_movement_bytes=110100480 total_nvlink_bytes=667828048 weight_to_token_ratio=0.197409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762735205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38776 dup_rows=21118 dup_rows_remote=18330 dedup_ratio=0.322235 router_tokens_per_expert=[515, 502, 506, 546, 516, 497, 518, 529, 492, 494, 514, 557, 513, 506, 500, 500, 510, 571, 498, 528, 520, 524, 501, 505, 481, 490, 509, 534, 481, 518, 503, 485, 508, 491, 531, 500, 540, 495, 505, 497, 521, 555, 463, 475, 536, 538, 498, 524, 494, 515, 490, 493, 519, 552, 520, 495, 487, 519, 514, 526, 520, 509, 535, 540] payload_rows_per_dst=[5642, 5614, 5514, 5456, 5574, 5542, 5540, 5536] dup_rows_per_dst=[2788, 2788, 2550, 2546, 2560, 2678, 2758, 2450] combine_rows_per_src=[5642, 5614, 5514, 5456, 5574, 5542, 5540, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44594 payload_rows_remote=39114 dup_rows=20942 dup_rows_remote=18518 dedup_ratio=0.319550 router_tokens_per_expert=[492, 524, 510, 490, 502, 526, 505, 524, 488, 473, 521, 537, 487, 503, 480, 463, 517, 508, 542, 509, 524, 514, 526, 561, 546, 500, 474, 498, 517, 538, 496, 535, 544, 550, 503, 522, 481, 537, 499, 516, 483, 509, 479, 497, 531, 522, 512, 530, 474, 491, 497, 491, 527, 541, 501, 529, 571, 488, 500, 523, 536, 521, 505, 528] payload_rows_per_dst=[5562, 5480, 5726, 5572, 5622, 5590, 5508, 5534] dup_rows_per_dst=[2584, 2424, 2676, 2636, 2682, 2536, 2594, 2810] combine_rows_per_src=[5562, 5480, 5726, 5572, 5622, 5590, 5508, 5534] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44730 payload_rows_remote=39138 dup_rows=20806 dup_rows_remote=18144 dedup_ratio=0.317474 router_tokens_per_expert=[518, 519, 514, 463, 521, 494, 519, 522, 540, 512, 478, 539, 499, 508, 487, 547, 506, 521, 502, 500, 505, 495, 530, 507, 507, 498, 512, 520, 520, 547, 487, 515, 537, 521, 529, 526, 480, 502, 505, 492, 481, 523, 529, 483, 513, 520, 526, 509, 512, 463, 558, 511, 537, 498, 532, 513, 522, 513, 503, 518, 561, 520, 487, 492] payload_rows_per_dst=[5554, 5624, 5390, 5742, 5612, 5564, 5652, 5592] dup_rows_per_dst=[2586, 2596, 2294, 2822, 2576, 2674, 2596, 2662] combine_rows_per_src=[5554, 5624, 5390, 5742, 5612, 5564, 5652, 5592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=39070 dup_rows=20918 dup_rows_remote=18318 dedup_ratio=0.319183 router_tokens_per_expert=[469, 471, 534, 503, 499, 516, 539, 512, 540, 491, 517, 537, 519, 525, 504, 528, 512, 509, 515, 505, 530, 499, 505, 509, 538, 522, 527, 498, 488, 498, 546, 492, 507, 507, 517, 534, 503, 503, 495, 510, 527, 539, 494, 523, 533, 473, 545, 520, 489, 493, 499, 510, 502, 532, 485, 564, 520, 488, 530, 492, 509, 502, 564, 462] payload_rows_per_dst=[5532, 5704, 5612, 5570, 5542, 5568, 5548, 5542] dup_rows_per_dst=[2554, 2618, 2556, 2648, 2610, 2740, 2600, 2592] combine_rows_per_src=[5532, 5704, 5612, 5570, 5542, 5568, 5548, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44382 payload_rows_remote=38684 dup_rows=21154 dup_rows_remote=18504 dedup_ratio=0.322784 router_tokens_per_expert=[485, 497, 515, 541, 491, 448, 532, 556, 521, 530, 531, 478, 495, 508, 513, 508, 562, 555, 472, 530, 497, 505, 530, 523, 488, 493, 519, 471, 512, 522, 511, 517, 532, 503, 509, 510, 515, 481, 537, 532, 501, 504, 534, 541, 499, 514, 515, 477, 462, 533, 493, 543, 500, 518, 525, 525, 486, 500, 491, 567, 492, 545, 524, 504] payload_rows_per_dst=[5562, 5428, 5698, 5474, 5622, 5532, 5570, 5496] dup_rows_per_dst=[2568, 2740, 2650, 2592, 2616, 2638, 2628, 2722] combine_rows_per_src=[5562, 5428, 5698, 5474, 5622, 5532, 5570, 5496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44360 payload_rows_remote=38866 dup_rows=21176 dup_rows_remote=18574 dedup_ratio=0.323120 router_tokens_per_expert=[495, 524, 474, 529, 545, 490, 517, 533, 479, 512, 518, 560, 478, 524, 524, 515, 502, 582, 553, 540, 507, 498, 486, 509, 508, 499, 488, 507, 509, 487, 510, 523, 505, 512, 494, 522, 528, 545, 500, 537, 505, 527, 499, 503, 505, 485, 521, 503, 507, 490, 479, 476, 515, 521, 529, 546, 500, 518, 491, 515, 510, 528, 532, 495] payload_rows_per_dst=[5512, 5524, 5620, 5490, 5660, 5494, 5544, 5516] dup_rows_per_dst=[2702, 2696, 2734, 2572, 2626, 2602, 2582, 2662] combine_rows_per_src=[5512, 5524, 5620, 5490, 5660, 5494, 5544, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38888 dup_rows=21084 dup_rows_remote=18412 dedup_ratio=0.321716 router_tokens_per_expert=[525, 517, 503, 530, 524, 517, 480, 503, 469, 522, 487, 491, 511, 479, 562, 528, 543, 476, 502, 485, 508, 502, 492, 536, 489, 533, 546, 507, 520, 515, 522, 486, 493, 531, 496, 507, 522, 540, 475, 516, 508, 497, 516, 464, 546, 528, 516, 536, 487, 518, 518, 521, 511, 526, 513, 481, 488, 518, 540, 584, 500, 493, 543, 526] payload_rows_per_dst=[5606, 5490, 5502, 5564, 5600, 5526, 5552, 5612] dup_rows_per_dst=[2592, 2608, 2586, 2672, 2560, 2696, 2598, 2772] combine_rows_per_src=[5606, 5490, 5502, 5564, 5600, 5526, 5552, 5612] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279781376 dispatch_meta_bytes=229848 combine_payload_bytes=279781376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280011224 ingress_bytes=389881856 link_bytes=389881856 token_bytes=559792600 weight_movement_bytes=110100480 total_nvlink_bytes=669893080 weight_to_token_ratio=0.196681 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780686721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228424 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278174792 ingress_bytes=388046848 link_bytes=388046848 token_bytes=556121160 weight_movement_bytes=110100480 total_nvlink_bytes=666221640 weight_to_token_ratio=0.197979 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748772914e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280369152 dispatch_meta_bytes=230528 combine_payload_bytes=280369152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280599680 ingress_bytes=390469632 link_bytes=390469632 token_bytes=560968832 weight_movement_bytes=110100480 total_nvlink_bytes=671069312 weight_to_token_ratio=0.196268 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790909113e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=38934 dup_rows=21024 dup_rows_remote=18286 dedup_ratio=0.320801 router_tokens_per_expert=[531, 510, 559, 538, 487, 509, 508, 514, 504, 490, 499, 500, 506, 512, 490, 505, 502, 512, 537, 528, 499, 520, 524, 536, 503, 497, 505, 491, 475, 517, 496, 477, 521, 515, 505, 532, 525, 526, 517, 553, 539, 529, 481, 521, 523, 492, 488, 511, 521, 495, 540, 514, 539, 530, 486, 524, 511, 514, 540, 482, 522, 515, 483, 493] payload_rows_per_dst=[5632, 5518, 5578, 5426, 5660, 5538, 5660, 5500] dup_rows_per_dst=[2680, 2494, 2738, 2496, 2728, 2630, 2638, 2620] combine_rows_per_src=[5632, 5518, 5578, 5426, 5660, 5538, 5660, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44646 payload_rows_remote=39118 dup_rows=20890 dup_rows_remote=18326 dedup_ratio=0.318756 router_tokens_per_expert=[514, 503, 469, 542, 501, 487, 529, 489, 546, 511, 507, 498, 514, 512, 503, 508, 496, 522, 497, 499, 501, 511, 504, 506, 475, 522, 525, 514, 538, 525, 542, 510, 532, 508, 515, 522, 516, 512, 470, 510, 513, 500, 502, 543, 525, 533, 519, 489, 498, 529, 525, 486, 516, 542, 507, 443, 532, 531, 529, 501, 538, 561, 498, 503] payload_rows_per_dst=[5550, 5642, 5474, 5652, 5514, 5618, 5528, 5668] dup_rows_per_dst=[2518, 2556, 2598, 2650, 2656, 2630, 2564, 2718] combine_rows_per_src=[5550, 5642, 5474, 5652, 5514, 5618, 5528, 5668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44594 payload_rows_remote=39114 dup_rows=20942 dup_rows_remote=18518 dedup_ratio=0.319550 router_tokens_per_expert=[492, 524, 510, 490, 502, 526, 505, 524, 488, 473, 521, 537, 487, 503, 480, 463, 517, 508, 542, 509, 524, 514, 526, 561, 546, 500, 474, 498, 517, 538, 496, 535, 544, 550, 503, 522, 481, 537, 499, 516, 483, 509, 479, 497, 531, 522, 512, 530, 474, 491, 497, 491, 527, 541, 501, 529, 571, 488, 500, 523, 536, 521, 505, 528] payload_rows_per_dst=[5562, 5480, 5726, 5572, 5622, 5590, 5508, 5534] dup_rows_per_dst=[2584, 2424, 2676, 2636, 2682, 2536, 2594, 2810] combine_rows_per_src=[5562, 5480, 5726, 5572, 5622, 5590, 5508, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38776 dup_rows=21118 dup_rows_remote=18330 dedup_ratio=0.322235 router_tokens_per_expert=[515, 502, 506, 546, 516, 497, 518, 529, 492, 494, 514, 557, 513, 506, 500, 500, 510, 571, 498, 528, 520, 524, 501, 505, 481, 490, 509, 534, 481, 518, 503, 485, 508, 491, 531, 500, 540, 495, 505, 497, 521, 555, 463, 475, 536, 538, 498, 524, 494, 515, 490, 493, 519, 552, 520, 495, 487, 519, 514, 526, 520, 509, 535, 540] payload_rows_per_dst=[5642, 5614, 5514, 5456, 5574, 5542, 5540, 5536] dup_rows_per_dst=[2788, 2788, 2550, 2546, 2560, 2678, 2758, 2450] combine_rows_per_src=[5642, 5614, 5514, 5456, 5574, 5542, 5540, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44360 payload_rows_remote=38866 dup_rows=21176 dup_rows_remote=18574 dedup_ratio=0.323120 router_tokens_per_expert=[495, 524, 474, 529, 545, 490, 517, 533, 479, 512, 518, 560, 478, 524, 524, 515, 502, 582, 553, 540, 507, 498, 486, 509, 508, 499, 488, 507, 509, 487, 510, 523, 505, 512, 494, 522, 528, 545, 500, 537, 505, 527, 499, 503, 505, 485, 521, 503, 507, 490, 479, 476, 515, 521, 529, 546, 500, 518, 491, 515, 510, 528, 532, 495] payload_rows_per_dst=[5512, 5524, 5620, 5490, 5660, 5494, 5544, 5516] dup_rows_per_dst=[2702, 2696, 2734, 2572, 2626, 2602, 2582, 2662] combine_rows_per_src=[5512, 5524, 5620, 5490, 5660, 5494, 5544, 5516] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280541184 dispatch_meta_bytes=229128 combine_payload_bytes=280541184 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280770312 ingress_bytes=610842624 link_bytes=610842624 token_bytes=561311496 weight_movement_bytes=330301440 total_nvlink_bytes=891612936 weight_to_token_ratio=0.588446 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062355789e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280053760 dispatch_meta_bytes=229552 combine_payload_bytes=280053760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280283312 ingress_bytes=390154240 link_bytes=390154240 token_bytes=560337072 weight_movement_bytes=110100480 total_nvlink_bytes=670437552 weight_to_token_ratio=0.196490 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785423927e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277286912 dispatch_meta_bytes=228752 combine_payload_bytes=277286912 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=277515664 ingress_bytes=827789312 link_bytes=827789312 token_bytes=554802576 weight_movement_bytes=550502400 total_nvlink_bytes=1105304976 weight_to_token_ratio=0.992249 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.439661762e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229200 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278978384 ingress_bytes=388849664 link_bytes=388849664 token_bytes=557727568 weight_movement_bytes=110100480 total_nvlink_bytes=667828048 weight_to_token_ratio=0.197409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762735205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=228880 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279307792 ingress_bytes=499279872 link_bytes=499279872 token_bytes=558386704 weight_movement_bytes=220200960 total_nvlink_bytes=778587664 weight_to_token_ratio=0.394352 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683298148e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280397824 dispatch_meta_bytes=229776 combine_payload_bytes=280397824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280627600 ingress_bytes=390498304 link_bytes=390498304 token_bytes=561025424 weight_movement_bytes=110100480 total_nvlink_bytes=671125904 weight_to_token_ratio=0.196249 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.791407766e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280369152 dispatch_meta_bytes=230528 combine_payload_bytes=280369152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280599680 ingress_bytes=390469632 link_bytes=390469632 token_bytes=560968832 weight_movement_bytes=110100480 total_nvlink_bytes=671069312 weight_to_token_ratio=0.196268 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790909113e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228424 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278174792 ingress_bytes=388046848 link_bytes=388046848 token_bytes=556121160 weight_movement_bytes=110100480 total_nvlink_bytes=666221640 weight_to_token_ratio=0.197979 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748772914e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278591488 dispatch_meta_bytes=229760 combine_payload_bytes=278591488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278821248 ingress_bytes=388691968 link_bytes=388691968 token_bytes=557412736 weight_movement_bytes=110100480 total_nvlink_bytes=667513216 weight_to_token_ratio=0.197521 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759992612e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38922 dup_rows=21114 dup_rows_remote=18604 dedup_ratio=0.322174 router_tokens_per_expert=[526, 532, 538, 489, 526, 505, 510, 519, 509, 527, 540, 492, 515, 491, 552, 497, 534, 464, 534, 494, 528, 497, 486, 535, 510, 510, 490, 515, 510, 514, 529, 545, 512, 487, 489, 476, 528, 530, 485, 498, 499, 458, 551, 479, 522, 503, 486, 510, 499, 512, 534, 517, 550, 519, 496, 506, 502, 532, 580, 500, 482, 484, 549, 530] payload_rows_per_dst=[5594, 5612, 5542, 5554, 5500, 5524, 5538, 5558] dup_rows_per_dst=[2696, 2634, 2602, 2692, 2510, 2492, 2728, 2760] combine_rows_per_src=[5594, 5612, 5542, 5554, 5500, 5524, 5538, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=230104 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279223000 ingress_bytes=609294336 link_bytes=609294336 token_bytes=558215896 weight_movement_bytes=330301440 total_nvlink_bytes=888517336 weight_to_token_ratio=0.591709 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059663062e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38834 dup_rows=21094 dup_rows_remote=18512 dedup_ratio=0.321869 router_tokens_per_expert=[538, 508, 487, 547, 517, 516, 501, 538, 507, 526, 521, 521, 489, 488, 524, 527, 500, 492, 528, 457, 537, 473, 535, 508, 513, 535, 524, 512, 486, 512, 515, 498, 507, 505, 518, 495, 510, 494, 528, 534, 485, 530, 542, 511, 530, 516, 518, 500, 526, 493, 492, 526, 502, 518, 516, 534, 512, 510, 489, 486, 523, 524, 509, 505] payload_rows_per_dst=[5546, 5556, 5504, 5608, 5532, 5618, 5610, 5468] dup_rows_per_dst=[2758, 2650, 2556, 2582, 2650, 2646, 2604, 2648] combine_rows_per_src=[5546, 5556, 5504, 5608, 5532, 5618, 5610, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38936 dup_rows=21052 dup_rows_remote=18408 dedup_ratio=0.321228 router_tokens_per_expert=[485, 524, 533, 531, 514, 513, 501, 507, 525, 535, 522, 493, 472, 489, 533, 549, 511, 504, 501, 517, 515, 520, 517, 505, 487, 497, 503, 541, 532, 502, 527, 459, 553, 511, 474, 545, 502, 519, 530, 504, 547, 513, 481, 485, 523, 527, 526, 495, 511, 531, 507, 506, 513, 528, 506, 538, 459, 529, 488, 562, 504, 511, 466, 510] payload_rows_per_dst=[5386, 5656, 5550, 5446, 5580, 5692, 5626, 5548] dup_rows_per_dst=[2540, 2578, 2630, 2516, 2652, 2792, 2700, 2644] combine_rows_per_src=[5386, 5656, 5550, 5446, 5580, 5692, 5626, 5548] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38922 dup_rows=21114 dup_rows_remote=18604 dedup_ratio=0.322174 router_tokens_per_expert=[526, 532, 538, 489, 526, 505, 510, 519, 509, 527, 540, 492, 515, 491, 552, 497, 534, 464, 534, 494, 528, 497, 486, 535, 510, 510, 490, 515, 510, 514, 529, 545, 512, 487, 489, 476, 528, 530, 485, 498, 499, 458, 551, 479, 522, 503, 486, 510, 499, 512, 534, 517, 550, 519, 496, 506, 502, 532, 580, 500, 482, 484, 549, 530] payload_rows_per_dst=[5594, 5612, 5542, 5554, 5500, 5524, 5538, 5558] dup_rows_per_dst=[2696, 2634, 2602, 2692, 2510, 2492, 2728, 2760] combine_rows_per_src=[5594, 5612, 5542, 5554, 5500, 5524, 5538, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278362112 dispatch_meta_bytes=229384 combine_payload_bytes=278362112 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278591496 ingress_bytes=498563072 link_bytes=498563072 token_bytes=556953608 weight_movement_bytes=220200960 total_nvlink_bytes=777154568 weight_to_token_ratio=0.395367 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670831817e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44382 payload_rows_remote=38684 dup_rows=21154 dup_rows_remote=18504 dedup_ratio=0.322784 router_tokens_per_expert=[485, 497, 515, 541, 491, 448, 532, 556, 521, 530, 531, 478, 495, 508, 513, 508, 562, 555, 472, 530, 497, 505, 530, 523, 488, 493, 519, 471, 512, 522, 511, 517, 532, 503, 509, 510, 515, 481, 537, 532, 501, 504, 534, 541, 499, 514, 515, 477, 462, 533, 493, 543, 500, 518, 525, 525, 486, 500, 491, 567, 492, 545, 524, 504] payload_rows_per_dst=[5562, 5428, 5698, 5474, 5622, 5532, 5570, 5496] dup_rows_per_dst=[2568, 2740, 2650, 2592, 2616, 2638, 2628, 2722] combine_rows_per_src=[5562, 5428, 5698, 5474, 5622, 5532, 5570, 5496] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=38904 dup_rows=21004 dup_rows_remote=18386 dedup_ratio=0.320496 router_tokens_per_expert=[485, 518, 515, 522, 514, 509, 477, 512, 521, 526, 506, 539, 508, 518, 524, 501, 545, 492, 519, 509, 515, 496, 470, 503, 467, 526, 479, 482, 562, 498, 519, 540, 516, 504, 502, 505, 516, 512, 506, 491, 531, 532, 531, 517, 566, 500, 485, 548, 545, 495, 496, 528, 494, 476, 512, 504, 525, 479, 522, 520, 498, 534, 518, 543] payload_rows_per_dst=[5628, 5570, 5764, 5596, 5460, 5628, 5522, 5364] dup_rows_per_dst=[2618, 2574, 2952, 2590, 2604, 2792, 2578, 2296] combine_rows_per_src=[5628, 5570, 5764, 5596, 5460, 5628, 5522, 5364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44524 payload_rows_remote=38868 dup_rows=21012 dup_rows_remote=18308 dedup_ratio=0.320618 router_tokens_per_expert=[487, 483, 488, 513, 508, 531, 563, 480, 528, 518, 507, 534, 500, 485, 501, 512, 519, 496, 500, 522, 505, 531, 546, 502, 502, 489, 517, 508, 561, 495, 533, 512, 478, 532, 498, 488, 509, 494, 526, 547, 554, 484, 541, 512, 523, 502, 495, 555, 540, 539, 500, 535, 505, 520, 528, 513, 502, 520, 492, 516, 498, 516, 445, 485] payload_rows_per_dst=[5562, 5580, 5630, 5572, 5506, 5570, 5656, 5448] dup_rows_per_dst=[2544, 2590, 2612, 2662, 2638, 2762, 2704, 2500] combine_rows_per_src=[5562, 5580, 5630, 5572, 5506, 5570, 5656, 5448] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44432 payload_rows_remote=39020 dup_rows=21104 dup_rows_remote=18518 dedup_ratio=0.322021 router_tokens_per_expert=[573, 489, 549, 481, 522, 526, 541, 532, 494, 506, 506, 482, 493, 505, 531, 482, 475, 508, 514, 481, 532, 511, 515, 541, 564, 529, 510, 504, 536, 480, 456, 522, 530, 514, 517, 525, 515, 492, 549, 504, 493, 488, 492, 511, 505, 496, 516, 482, 486, 498, 511, 538, 507, 498, 518, 540, 521, 475, 513, 536, 524, 523, 513, 548] payload_rows_per_dst=[5680, 5412, 5514, 5524, 5676, 5468, 5544, 5614] dup_rows_per_dst=[2746, 2586, 2640, 2678, 2616, 2498, 2648, 2692] combine_rows_per_src=[5680, 5412, 5514, 5524, 5676, 5468, 5544, 5614] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38936 dup_rows=21052 dup_rows_remote=18408 dedup_ratio=0.321228 router_tokens_per_expert=[485, 524, 533, 531, 514, 513, 501, 507, 525, 535, 522, 493, 472, 489, 533, 549, 511, 504, 501, 517, 515, 520, 517, 505, 487, 497, 503, 541, 532, 502, 527, 459, 553, 511, 474, 545, 502, 519, 530, 504, 547, 513, 481, 485, 523, 527, 526, 495, 511, 531, 507, 506, 513, 528, 506, 538, 459, 529, 488, 562, 504, 511, 466, 510] payload_rows_per_dst=[5386, 5656, 5550, 5446, 5580, 5692, 5626, 5548] dup_rows_per_dst=[2540, 2578, 2630, 2516, 2652, 2792, 2700, 2644] combine_rows_per_src=[5386, 5656, 5550, 5446, 5580, 5692, 5626, 5548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44646 payload_rows_remote=39118 dup_rows=20890 dup_rows_remote=18326 dedup_ratio=0.318756 router_tokens_per_expert=[514, 503, 469, 542, 501, 487, 529, 489, 546, 511, 507, 498, 514, 512, 503, 508, 496, 522, 497, 499, 501, 511, 504, 506, 475, 522, 525, 514, 538, 525, 542, 510, 532, 508, 515, 522, 516, 512, 470, 510, 513, 500, 502, 543, 525, 533, 519, 489, 498, 529, 525, 486, 516, 542, 507, 443, 532, 531, 529, 501, 538, 561, 498, 503] payload_rows_per_dst=[5550, 5642, 5474, 5652, 5514, 5618, 5528, 5668] dup_rows_per_dst=[2518, 2556, 2598, 2650, 2656, 2630, 2564, 2718] combine_rows_per_src=[5550, 5642, 5474, 5652, 5514, 5618, 5528, 5668] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=230104 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279223000 ingress_bytes=609294336 link_bytes=609294336 token_bytes=558215896 weight_movement_bytes=330301440 total_nvlink_bytes=888517336 weight_to_token_ratio=0.591709 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059663062e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278362112 dispatch_meta_bytes=229384 combine_payload_bytes=278362112 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278591496 ingress_bytes=498563072 link_bytes=498563072 token_bytes=556953608 weight_movement_bytes=220200960 total_nvlink_bytes=777154568 weight_to_token_ratio=0.395367 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670831817e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44674 payload_rows_remote=39108 dup_rows=20862 dup_rows_remote=18366 dedup_ratio=0.318329 router_tokens_per_expert=[508, 526, 497, 545, 499, 482, 492, 478, 518, 524, 496, 539, 515, 523, 490, 536, 531, 509, 548, 512, 476, 507, 507, 484, 532, 530, 540, 501, 535, 490, 487, 515, 536, 496, 480, 551, 515, 520, 555, 527, 511, 479, 514, 495, 504, 535, 481, 512, 503, 527, 483, 518, 499, 504, 532, 505, 474, 546, 498, 544, 528, 502, 507, 515] payload_rows_per_dst=[5494, 5596, 5580, 5622, 5650, 5566, 5570, 5596] dup_rows_per_dst=[2560, 2686, 2568, 2638, 2710, 2496, 2572, 2632] combine_rows_per_src=[5494, 5596, 5580, 5622, 5650, 5566, 5570, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279093248 dispatch_meta_bytes=229376 combine_payload_bytes=279093248 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279322624 ingress_bytes=499294208 link_bytes=499294208 token_bytes=558415872 weight_movement_bytes=220200960 total_nvlink_bytes=778616832 weight_to_token_ratio=0.394331 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683547475e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38962 dup_rows=21050 dup_rows_remote=18512 dedup_ratio=0.321198 router_tokens_per_expert=[511, 527, 515, 529, 514, 509, 499, 507, 496, 538, 511, 511, 532, 529, 510, 487, 493, 520, 515, 515, 483, 483, 516, 506, 485, 514, 513, 582, 440, 507, 528, 525, 531, 545, 435, 522, 508, 518, 539, 515, 531, 459, 512, 510, 525, 482, 491, 528, 521, 485, 521, 498, 507, 509, 548, 513, 469, 541, 549, 522, 518, 532, 530, 504] payload_rows_per_dst=[5622, 5528, 5524, 5594, 5574, 5448, 5560, 5636] dup_rows_per_dst=[2600, 2700, 2538, 2594, 2652, 2628, 2644, 2694] combine_rows_per_src=[5622, 5528, 5524, 5594, 5574, 5448, 5560, 5636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229160 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279093032 ingress_bytes=499064832 link_bytes=499064832 token_bytes=557956904 weight_movement_bytes=220200960 total_nvlink_bytes=778157864 weight_to_token_ratio=0.394656 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679558249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44634 payload_rows_remote=39086 dup_rows=20902 dup_rows_remote=18326 dedup_ratio=0.318939 router_tokens_per_expert=[471, 487, 529, 526, 495, 530, 513, 519, 527, 512, 515, 530, 510, 509, 477, 480, 525, 535, 519, 502, 509, 524, 478, 494, 523, 510, 487, 511, 518, 498, 530, 542, 480, 531, 505, 515, 512, 519, 487, 513, 515, 505, 511, 502, 544, 489, 498, 557, 515, 489, 521, 531, 524, 522, 512, 535, 527, 502, 507, 511, 528, 505, 505, 516] payload_rows_per_dst=[5656, 5578, 5526, 5590, 5548, 5544, 5590, 5602] dup_rows_per_dst=[2484, 2542, 2646, 2648, 2576, 2698, 2708, 2600] combine_rows_per_src=[5656, 5578, 5526, 5590, 5548, 5544, 5590, 5602] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277286912 dispatch_meta_bytes=228752 combine_payload_bytes=277286912 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=277515664 ingress_bytes=827789312 link_bytes=827789312 token_bytes=554802576 weight_movement_bytes=550502400 total_nvlink_bytes=1105304976 weight_to_token_ratio=0.992249 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.439661762e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229160 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279093032 ingress_bytes=499064832 link_bytes=499064832 token_bytes=557956904 weight_movement_bytes=220200960 total_nvlink_bytes=778157864 weight_to_token_ratio=0.394656 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679558249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=228704 combine_payload_bytes=278605824 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278834528 ingress_bytes=719007744 link_bytes=719007744 token_bytes=557440352 weight_movement_bytes=440401920 total_nvlink_bytes=997842272 weight_to_token_ratio=0.790043 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250472723e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279695360 dispatch_meta_bytes=230152 combine_payload_bytes=279695360 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279925512 ingress_bytes=499896320 link_bytes=499896320 token_bytes=559620872 weight_movement_bytes=220200960 total_nvlink_bytes=779821832 weight_to_token_ratio=0.393482 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694019193e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279093248 dispatch_meta_bytes=229376 combine_payload_bytes=279093248 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279322624 ingress_bytes=499294208 link_bytes=499294208 token_bytes=558415872 weight_movement_bytes=220200960 total_nvlink_bytes=778616832 weight_to_token_ratio=0.394331 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683547475e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280397824 dispatch_meta_bytes=229776 combine_payload_bytes=280397824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280627600 ingress_bytes=390498304 link_bytes=390498304 token_bytes=561025424 weight_movement_bytes=110100480 total_nvlink_bytes=671125904 weight_to_token_ratio=0.196249 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.791407766e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280326144 dispatch_meta_bytes=229896 combine_payload_bytes=280326144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280556040 ingress_bytes=390426624 link_bytes=390426624 token_bytes=560882184 weight_movement_bytes=110100480 total_nvlink_bytes=670982664 weight_to_token_ratio=0.196299 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790161133e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280168448 dispatch_meta_bytes=229648 combine_payload_bytes=280168448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280398096 ingress_bytes=390268928 link_bytes=390268928 token_bytes=560566544 weight_movement_bytes=110100480 total_nvlink_bytes=670667024 weight_to_token_ratio=0.196409 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787418540e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=39004 dup_rows=20992 dup_rows_remote=18498 dedup_ratio=0.320312 router_tokens_per_expert=[523, 507, 560, 477, 526, 493, 522, 521, 540, 523, 538, 494, 497, 544, 532, 500, 511, 493, 503, 492, 522, 526, 482, 486, 510, 497, 512, 500, 517, 492, 480, 509, 544, 482, 488, 517, 537, 527, 552, 512, 509, 508, 473, 499, 519, 511, 541, 546, 512, 523, 515, 493, 524, 494, 545, 540, 480, 515, 494, 516, 506, 496, 498, 523] payload_rows_per_dst=[5620, 5612, 5494, 5540, 5570, 5584, 5650, 5474] dup_rows_per_dst=[2638, 2724, 2536, 2494, 2748, 2628, 2642, 2582] combine_rows_per_src=[5620, 5612, 5494, 5540, 5570, 5584, 5650, 5474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=229896 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279509512 ingress_bytes=389380096 link_bytes=389380096 token_bytes=558789128 weight_movement_bytes=110100480 total_nvlink_bytes=668889608 weight_to_token_ratio=0.197034 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771960290e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=38940 dup_rows=20946 dup_rows_remote=18318 dedup_ratio=0.319611 router_tokens_per_expert=[493, 482, 517, 461, 466, 501, 513, 503, 522, 490, 509, 505, 509, 536, 528, 513, 533, 504, 552, 492, 522, 496, 517, 521, 492, 504, 516, 502, 529, 511, 488, 515, 501, 513, 513, 521, 482, 530, 493, 504, 515, 509, 551, 508, 515, 540, 499, 544, 550, 501, 550, 513, 513, 495, 511, 516, 513, 526, 504, 558, 522, 529, 495, 492] payload_rows_per_dst=[5404, 5638, 5592, 5564, 5558, 5572, 5612, 5650] dup_rows_per_dst=[2468, 2586, 2682, 2550, 2556, 2790, 2686, 2628] combine_rows_per_src=[5404, 5638, 5592, 5564, 5558, 5572, 5612, 5650] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44634 payload_rows_remote=39086 dup_rows=20902 dup_rows_remote=18326 dedup_ratio=0.318939 router_tokens_per_expert=[471, 487, 529, 526, 495, 530, 513, 519, 527, 512, 515, 530, 510, 509, 477, 480, 525, 535, 519, 502, 509, 524, 478, 494, 523, 510, 487, 511, 518, 498, 530, 542, 480, 531, 505, 515, 512, 519, 487, 513, 515, 505, 511, 502, 544, 489, 498, 557, 515, 489, 521, 531, 524, 522, 512, 535, 527, 502, 507, 511, 528, 505, 505, 516] payload_rows_per_dst=[5656, 5578, 5526, 5590, 5548, 5544, 5590, 5602] dup_rows_per_dst=[2484, 2542, 2646, 2648, 2576, 2698, 2708, 2600] combine_rows_per_src=[5656, 5578, 5526, 5590, 5548, 5544, 5590, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279580672 dispatch_meta_bytes=230008 combine_payload_bytes=279580672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279810680 ingress_bytes=389681152 link_bytes=389681152 token_bytes=559391352 weight_movement_bytes=110100480 total_nvlink_bytes=669491832 weight_to_token_ratio=0.196822 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777196149e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=38730 dup_rows=21044 dup_rows_remote=18154 dedup_ratio=0.321106 router_tokens_per_expert=[508, 530, 529, 533, 489, 482, 517, 508, 530, 507, 498, 520, 522, 507, 556, 500, 497, 508, 514, 535, 520, 523, 506, 489, 543, 505, 521, 495, 512, 513, 504, 518, 513, 483, 518, 478, 501, 514, 508, 510, 513, 486, 500, 518, 490, 495, 522, 550, 506, 523, 541, 487, 493, 499, 530, 540, 526, 506, 506, 541, 493, 505, 524, 510] payload_rows_per_dst=[5762, 5574, 5658, 5592, 5590, 5446, 5312, 5558] dup_rows_per_dst=[2890, 2720, 2628, 2842, 2644, 2424, 2274, 2622] combine_rows_per_src=[5762, 5574, 5658, 5592, 5590, 5446, 5312, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38962 dup_rows=21050 dup_rows_remote=18512 dedup_ratio=0.321198 router_tokens_per_expert=[511, 527, 515, 529, 514, 509, 499, 507, 496, 538, 511, 511, 532, 529, 510, 487, 493, 520, 515, 515, 483, 483, 516, 506, 485, 514, 513, 582, 440, 507, 528, 525, 531, 545, 435, 522, 508, 518, 539, 515, 531, 459, 512, 510, 525, 482, 491, 528, 521, 485, 521, 498, 507, 509, 548, 513, 469, 541, 549, 522, 518, 532, 530, 504] payload_rows_per_dst=[5622, 5528, 5524, 5594, 5574, 5448, 5560, 5636] dup_rows_per_dst=[2600, 2700, 2538, 2594, 2652, 2628, 2644, 2694] combine_rows_per_src=[5622, 5528, 5524, 5594, 5574, 5448, 5560, 5636] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38872 dup_rows=21072 dup_rows_remote=18460 dedup_ratio=0.321533 router_tokens_per_expert=[507, 534, 512, 509, 483, 496, 505, 482, 543, 522, 491, 504, 488, 539, 487, 528, 485, 507, 491, 537, 492, 528, 507, 546, 541, 484, 516, 543, 486, 499, 523, 490, 521, 525, 525, 519, 517, 509, 518, 532, 495, 549, 524, 470, 489, 482, 538, 494, 528, 554, 517, 516, 500, 528, 527, 503, 520, 484, 508, 499, 482, 517, 531, 542] payload_rows_per_dst=[5474, 5592, 5566, 5624, 5632, 5416, 5622, 5538] dup_rows_per_dst=[2582, 2612, 2620, 2540, 2700, 2666, 2724, 2628] combine_rows_per_src=[5474, 5592, 5566, 5624, 5632, 5416, 5622, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280168448 dispatch_meta_bytes=229648 combine_payload_bytes=280168448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280398096 ingress_bytes=390268928 link_bytes=390268928 token_bytes=560566544 weight_movement_bytes=110100480 total_nvlink_bytes=670667024 weight_to_token_ratio=0.196409 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787418540e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=38940 dup_rows=20946 dup_rows_remote=18318 dedup_ratio=0.319611 router_tokens_per_expert=[493, 482, 517, 461, 466, 501, 513, 503, 522, 490, 509, 505, 509, 536, 528, 513, 533, 504, 552, 492, 522, 496, 517, 521, 492, 504, 516, 502, 529, 511, 488, 515, 501, 513, 513, 521, 482, 530, 493, 504, 515, 509, 551, 508, 515, 540, 499, 544, 550, 501, 550, 513, 513, 495, 511, 516, 513, 526, 504, 558, 522, 529, 495, 492] payload_rows_per_dst=[5404, 5638, 5592, 5564, 5558, 5572, 5612, 5650] dup_rows_per_dst=[2468, 2586, 2682, 2550, 2556, 2790, 2686, 2628] combine_rows_per_src=[5404, 5638, 5592, 5564, 5558, 5572, 5612, 5650] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44524 payload_rows_remote=38868 dup_rows=21012 dup_rows_remote=18308 dedup_ratio=0.320618 router_tokens_per_expert=[487, 483, 488, 513, 508, 531, 563, 480, 528, 518, 507, 534, 500, 485, 501, 512, 519, 496, 500, 522, 505, 531, 546, 502, 502, 489, 517, 508, 561, 495, 533, 512, 478, 532, 498, 488, 509, 494, 526, 547, 554, 484, 541, 512, 523, 502, 495, 555, 540, 539, 500, 535, 505, 520, 528, 513, 502, 520, 492, 516, 498, 516, 445, 485] payload_rows_per_dst=[5562, 5580, 5630, 5572, 5506, 5570, 5656, 5448] dup_rows_per_dst=[2544, 2590, 2612, 2662, 2638, 2762, 2704, 2500] combine_rows_per_src=[5562, 5580, 5630, 5572, 5506, 5570, 5656, 5448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=39046 dup_rows=21014 dup_rows_remote=18524 dedup_ratio=0.320648 router_tokens_per_expert=[475, 512, 503, 482, 526, 526, 526, 518, 545, 517, 497, 508, 515, 488, 519, 552, 520, 513, 498, 502, 513, 547, 528, 476, 496, 519, 538, 530, 502, 504, 533, 506, 464, 542, 516, 558, 544, 488, 509, 499, 497, 533, 555, 517, 514, 526, 521, 513, 492, 525, 459, 526, 520, 481, 496, 484, 539, 501, 501, 518, 495, 489, 522, 490] payload_rows_per_dst=[5544, 5594, 5494, 5602, 5604, 5616, 5476, 5592] dup_rows_per_dst=[2592, 2688, 2700, 2654, 2636, 2736, 2490, 2518] combine_rows_per_src=[5544, 5594, 5494, 5602, 5604, 5616, 5476, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=38730 dup_rows=21044 dup_rows_remote=18154 dedup_ratio=0.321106 router_tokens_per_expert=[508, 530, 529, 533, 489, 482, 517, 508, 530, 507, 498, 520, 522, 507, 556, 500, 497, 508, 514, 535, 520, 523, 506, 489, 543, 505, 521, 495, 512, 513, 504, 518, 513, 483, 518, 478, 501, 514, 508, 510, 513, 486, 500, 518, 490, 495, 522, 550, 506, 523, 541, 487, 493, 499, 530, 540, 526, 506, 506, 541, 493, 505, 524, 510] payload_rows_per_dst=[5762, 5574, 5658, 5592, 5590, 5446, 5312, 5558] dup_rows_per_dst=[2890, 2720, 2628, 2842, 2644, 2424, 2274, 2622] combine_rows_per_src=[5762, 5574, 5658, 5592, 5590, 5446, 5312, 5558] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38880 dup_rows=21118 dup_rows_remote=18400 dedup_ratio=0.322235 router_tokens_per_expert=[509, 520, 499, 543, 503, 525, 500, 499, 536, 496, 549, 517, 508, 503, 488, 487, 486, 519, 536, 516, 476, 515, 530, 481, 489, 536, 487, 497, 528, 506, 537, 478, 506, 536, 500, 535, 523, 493, 499, 532, 513, 544, 511, 529, 505, 523, 478, 525, 522, 486, 539, 493, 489, 521, 541, 491, 522, 513, 500, 518, 515, 507, 533, 527] payload_rows_per_dst=[5546, 5510, 5520, 5494, 5676, 5538, 5554, 5580] dup_rows_per_dst=[2650, 2658, 2598, 2622, 2572, 2718, 2610, 2690] combine_rows_per_src=[5546, 5510, 5520, 5494, 5676, 5538, 5554, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=229032 combine_payload_bytes=279121920 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279350952 ingress_bytes=499322880 link_bytes=499322880 token_bytes=558472872 weight_movement_bytes=220200960 total_nvlink_bytes=778673832 weight_to_token_ratio=0.394291 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684046128e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38862 dup_rows=21050 dup_rows_remote=18446 dedup_ratio=0.321198 router_tokens_per_expert=[523, 512, 525, 536, 508, 503, 519, 502, 480, 506, 475, 478, 489, 542, 524, 516, 513, 502, 484, 500, 537, 524, 531, 523, 493, 535, 530, 536, 508, 521, 482, 496, 542, 498, 492, 486, 491, 507, 505, 490, 468, 497, 538, 516, 491, 553, 516, 511, 523, 514, 473, 496, 528, 512, 524, 526, 522, 541, 514, 528, 529, 540, 533, 511] payload_rows_per_dst=[5550, 5502, 5624, 5538, 5508, 5542, 5568, 5654] dup_rows_per_dst=[2706, 2518, 2604, 2664, 2514, 2638, 2624, 2782] combine_rows_per_src=[5550, 5502, 5624, 5538, 5508, 5542, 5568, 5654] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277616640 dispatch_meta_bytes=227536 combine_payload_bytes=277616640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277844176 ingress_bytes=387717120 link_bytes=387717120 token_bytes=555460816 weight_movement_bytes=110100480 total_nvlink_bytes=665561296 weight_to_token_ratio=0.198215 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.743038402e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38866 dup_rows=21094 dup_rows_remote=18528 dedup_ratio=0.321869 router_tokens_per_expert=[527, 531, 509, 502, 518, 535, 486, 504, 496, 540, 498, 481, 518, 514, 494, 520, 521, 558, 486, 500, 497, 511, 506, 546, 523, 481, 495, 520, 517, 505, 515, 506, 490, 531, 522, 537, 495, 471, 510, 515, 552, 537, 479, 500, 526, 526, 535, 539, 479, 536, 498, 530, 511, 500, 509, 527, 483, 522, 527, 527, 502, 492, 507, 493] payload_rows_per_dst=[5640, 5498, 5542, 5542, 5576, 5614, 5514, 5516] dup_rows_per_dst=[2584, 2624, 2708, 2582, 2566, 2774, 2666, 2590] combine_rows_per_src=[5640, 5498, 5542, 5542, 5576, 5614, 5514, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278634496 dispatch_meta_bytes=229328 combine_payload_bytes=278634496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278863824 ingress_bytes=388734976 link_bytes=388734976 token_bytes=557498320 weight_movement_bytes=110100480 total_nvlink_bytes=667598800 weight_to_token_ratio=0.197490 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760740592e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=229032 combine_payload_bytes=279121920 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279350952 ingress_bytes=499322880 link_bytes=499322880 token_bytes=558472872 weight_movement_bytes=220200960 total_nvlink_bytes=778673832 weight_to_token_ratio=0.394291 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684046128e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=228704 combine_payload_bytes=278605824 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278834528 ingress_bytes=719007744 link_bytes=719007744 token_bytes=557440352 weight_movement_bytes=440401920 total_nvlink_bytes=997842272 weight_to_token_ratio=0.790043 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250472723e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=229896 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279509512 ingress_bytes=389380096 link_bytes=389380096 token_bytes=558789128 weight_movement_bytes=110100480 total_nvlink_bytes=668889608 weight_to_token_ratio=0.197034 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771960290e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=230280 combine_payload_bytes=279881728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280112008 ingress_bytes=389982208 link_bytes=389982208 token_bytes=559993736 weight_movement_bytes=110100480 total_nvlink_bytes=670094216 weight_to_token_ratio=0.196610 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782432008e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229120 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278920960 ingress_bytes=608993280 link_bytes=608993280 token_bytes=557612800 weight_movement_bytes=330301440 total_nvlink_bytes=887914240 weight_to_token_ratio=0.592349 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059139476e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277616640 dispatch_meta_bytes=227536 combine_payload_bytes=277616640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277844176 ingress_bytes=387717120 link_bytes=387717120 token_bytes=555460816 weight_movement_bytes=110100480 total_nvlink_bytes=665561296 weight_to_token_ratio=0.198215 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.743038402e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278591488 dispatch_meta_bytes=229576 combine_payload_bytes=278591488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278821064 ingress_bytes=388691968 link_bytes=388691968 token_bytes=557412552 weight_movement_bytes=110100480 total_nvlink_bytes=667513032 weight_to_token_ratio=0.197521 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759992612e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38774 dup_rows=21144 dup_rows_remote=18378 dedup_ratio=0.322632 router_tokens_per_expert=[514, 501, 486, 533, 522, 509, 490, 468, 508, 516, 456, 469, 531, 528, 546, 504, 474, 468, 540, 510, 532, 533, 500, 546, 526, 512, 505, 544, 532, 510, 512, 551, 505, 535, 509, 509, 535, 508, 515, 508, 536, 500, 510, 450, 516, 534, 521, 499, 509, 486, 509, 471, 480, 473, 546, 509, 531, 528, 519, 513, 570, 518, 528, 512] payload_rows_per_dst=[5528, 5502, 5564, 5618, 5600, 5536, 5406, 5638] dup_rows_per_dst=[2518, 2614, 2642, 2766, 2648, 2596, 2560, 2800] combine_rows_per_src=[5528, 5502, 5564, 5618, 5600, 5536, 5406, 5638] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278562816 dispatch_meta_bytes=229232 combine_payload_bytes=278562816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278792048 ingress_bytes=388663296 link_bytes=388663296 token_bytes=557354864 weight_movement_bytes=110100480 total_nvlink_bytes=667455344 weight_to_token_ratio=0.197541 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759493959e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38866 dup_rows=21094 dup_rows_remote=18528 dedup_ratio=0.321869 router_tokens_per_expert=[527, 531, 509, 502, 518, 535, 486, 504, 496, 540, 498, 481, 518, 514, 494, 520, 521, 558, 486, 500, 497, 511, 506, 546, 523, 481, 495, 520, 517, 505, 515, 506, 490, 531, 522, 537, 495, 471, 510, 515, 552, 537, 479, 500, 526, 526, 535, 539, 479, 536, 498, 530, 511, 500, 509, 527, 483, 522, 527, 527, 502, 492, 507, 493] payload_rows_per_dst=[5640, 5498, 5542, 5542, 5576, 5614, 5514, 5516] dup_rows_per_dst=[2584, 2624, 2708, 2582, 2566, 2774, 2666, 2590] combine_rows_per_src=[5640, 5498, 5542, 5542, 5576, 5614, 5514, 5516] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38808 dup_rows=21170 dup_rows_remote=18650 dedup_ratio=0.323029 router_tokens_per_expert=[491, 508, 534, 537, 520, 526, 526, 520, 485, 500, 514, 537, 475, 521, 515, 528, 522, 499, 504, 488, 519, 505, 550, 491, 489, 523, 544, 523, 499, 537, 531, 511, 523, 510, 505, 522, 528, 524, 505, 504, 539, 496, 489, 470, 539, 505, 518, 531, 523, 516, 506, 523, 521, 498, 486, 495, 514, 515, 517, 513, 472, 469, 496, 524] payload_rows_per_dst=[5578, 5462, 5512, 5538, 5682, 5294, 5742, 5558] dup_rows_per_dst=[2746, 2676, 2644, 2512, 2836, 2280, 2956, 2520] combine_rows_per_src=[5578, 5462, 5512, 5538, 5682, 5294, 5742, 5558] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277932032 dispatch_meta_bytes=228608 combine_payload_bytes=277932032 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278160640 ingress_bytes=608233472 link_bytes=608233472 token_bytes=556092672 weight_movement_bytes=330301440 total_nvlink_bytes=886394112 weight_to_token_ratio=0.593968 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057818045e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38788 dup_rows=21070 dup_rows_remote=18318 dedup_ratio=0.321503 router_tokens_per_expert=[520, 496, 537, 500, 517, 529, 488, 543, 492, 498, 496, 522, 474, 515, 538, 543, 534, 478, 476, 522, 501, 517, 502, 546, 525, 475, 556, 526, 531, 534, 514, 496, 517, 522, 499, 481, 502, 504, 508, 486, 531, 477, 543, 520, 519, 552, 526, 520, 497, 527, 503, 501, 520, 486, 547, 509, 460, 531, 509, 502, 478, 487, 510, 553] payload_rows_per_dst=[5678, 5442, 5594, 5576, 5540, 5562, 5576, 5498] dup_rows_per_dst=[2752, 2548, 2684, 2612, 2498, 2810, 2604, 2562] combine_rows_per_src=[5678, 5442, 5594, 5576, 5540, 5562, 5576, 5498] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278591488 dispatch_meta_bytes=229576 combine_payload_bytes=278591488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278821064 ingress_bytes=388691968 link_bytes=388691968 token_bytes=557412552 weight_movement_bytes=110100480 total_nvlink_bytes=667513032 weight_to_token_ratio=0.197521 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759992612e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44758 payload_rows_remote=39118 dup_rows=20778 dup_rows_remote=18140 dedup_ratio=0.317047 router_tokens_per_expert=[513, 477, 516, 533, 456, 526, 531, 473, 513, 505, 517, 501, 507, 534, 530, 532, 462, 533, 525, 515, 486, 499, 511, 532, 482, 535, 497, 526, 507, 506, 518, 501, 492, 481, 510, 510, 511, 534, 503, 480, 507, 524, 526, 506, 466, 517, 490, 531, 525, 509, 510, 522, 549, 548, 524, 500, 533, 544, 532, 488, 510, 518, 538, 531] payload_rows_per_dst=[5512, 5640, 5550, 5586, 5558, 5578, 5630, 5704] dup_rows_per_dst=[2538, 2638, 2576, 2558, 2484, 2556, 2744, 2684] combine_rows_per_src=[5512, 5640, 5550, 5586, 5558, 5578, 5630, 5704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38808 dup_rows=21170 dup_rows_remote=18650 dedup_ratio=0.323029 router_tokens_per_expert=[491, 508, 534, 537, 520, 526, 526, 520, 485, 500, 514, 537, 475, 521, 515, 528, 522, 499, 504, 488, 519, 505, 550, 491, 489, 523, 544, 523, 499, 537, 531, 511, 523, 510, 505, 522, 528, 524, 505, 504, 539, 496, 489, 470, 539, 505, 518, 531, 523, 516, 506, 523, 521, 498, 486, 495, 514, 515, 517, 513, 472, 469, 496, 524] payload_rows_per_dst=[5578, 5462, 5512, 5538, 5682, 5294, 5742, 5558] dup_rows_per_dst=[2746, 2676, 2644, 2512, 2836, 2280, 2956, 2520] combine_rows_per_src=[5578, 5462, 5512, 5538, 5682, 5294, 5742, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=39046 dup_rows=21014 dup_rows_remote=18524 dedup_ratio=0.320648 router_tokens_per_expert=[475, 512, 503, 482, 526, 526, 526, 518, 545, 517, 497, 508, 515, 488, 519, 552, 520, 513, 498, 502, 513, 547, 528, 476, 496, 519, 538, 530, 502, 504, 533, 506, 464, 542, 516, 558, 544, 488, 509, 499, 497, 533, 555, 517, 514, 526, 521, 513, 492, 525, 459, 526, 520, 481, 496, 484, 539, 501, 501, 518, 495, 489, 522, 490] payload_rows_per_dst=[5544, 5594, 5494, 5602, 5604, 5616, 5476, 5592] dup_rows_per_dst=[2592, 2688, 2700, 2654, 2636, 2736, 2490, 2518] combine_rows_per_src=[5544, 5594, 5494, 5602, 5604, 5616, 5476, 5592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38862 dup_rows=21050 dup_rows_remote=18446 dedup_ratio=0.321198 router_tokens_per_expert=[523, 512, 525, 536, 508, 503, 519, 502, 480, 506, 475, 478, 489, 542, 524, 516, 513, 502, 484, 500, 537, 524, 531, 523, 493, 535, 530, 536, 508, 521, 482, 496, 542, 498, 492, 486, 491, 507, 505, 490, 468, 497, 538, 516, 491, 553, 516, 511, 523, 514, 473, 496, 528, 512, 524, 526, 522, 541, 514, 528, 529, 540, 533, 511] payload_rows_per_dst=[5550, 5502, 5624, 5538, 5508, 5542, 5568, 5654] dup_rows_per_dst=[2706, 2518, 2604, 2664, 2514, 2638, 2624, 2782] combine_rows_per_src=[5550, 5502, 5624, 5538, 5508, 5542, 5568, 5654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44606 payload_rows_remote=38938 dup_rows=20930 dup_rows_remote=18158 dedup_ratio=0.319366 router_tokens_per_expert=[484, 522, 496, 488, 516, 525, 503, 552, 500, 515, 499, 521, 483, 505, 511, 522, 496, 501, 542, 526, 539, 528, 503, 498, 502, 494, 476, 501, 523, 516, 516, 527, 485, 483, 510, 509, 538, 472, 511, 521, 561, 513, 513, 523, 459, 532, 505, 503, 529, 541, 542, 534, 529, 506, 534, 505, 519, 539, 461, 494, 496, 504, 521, 546] payload_rows_per_dst=[5588, 5518, 5682, 5516, 5552, 5592, 5668, 5490] dup_rows_per_dst=[2584, 2594, 2584, 2594, 2506, 2626, 2772, 2670] combine_rows_per_src=[5588, 5518, 5682, 5516, 5552, 5592, 5668, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278175744 dispatch_meta_bytes=229832 combine_payload_bytes=278175744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278405576 ingress_bytes=388276224 link_bytes=388276224 token_bytes=556581320 weight_movement_bytes=110100480 total_nvlink_bytes=666681800 weight_to_token_ratio=0.197816 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752762140e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38856 dup_rows=21026 dup_rows_remote=18456 dedup_ratio=0.320831 router_tokens_per_expert=[546, 500, 511, 504, 496, 497, 528, 509, 506, 537, 518, 513, 539, 524, 498, 501, 543, 492, 544, 537, 531, 500, 502, 508, 503, 535, 503, 517, 499, 500, 519, 547, 539, 506, 507, 499, 519, 535, 520, 502, 536, 497, 516, 528, 511, 466, 554, 504, 469, 487, 502, 499, 503, 515, 456, 490, 522, 540, 514, 501, 495, 514, 511, 504] payload_rows_per_dst=[5540, 5608, 5584, 5620, 5574, 5654, 5416, 5514] dup_rows_per_dst=[2642, 2664, 2730, 2626, 2680, 2570, 2426, 2688] combine_rows_per_src=[5540, 5608, 5584, 5620, 5574, 5654, 5416, 5514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38990 dup_rows=21026 dup_rows_remote=18534 dedup_ratio=0.320831 router_tokens_per_expert=[471, 510, 526, 512, 517, 510, 492, 534, 537, 507, 501, 505, 546, 559, 509, 505, 504, 483, 505, 517, 493, 496, 500, 508, 495, 520, 497, 500, 543, 495, 511, 511, 517, 488, 513, 536, 483, 516, 496, 478, 469, 528, 505, 531, 569, 527, 552, 499, 512, 528, 504, 527, 529, 491, 548, 526, 526, 512, 522, 523, 518, 480, 475, 521] payload_rows_per_dst=[5506, 5634, 5520, 5500, 5474, 5640, 5654, 5582] dup_rows_per_dst=[2638, 2704, 2492, 2644, 2580, 2720, 2676, 2572] combine_rows_per_src=[5506, 5634, 5520, 5500, 5474, 5640, 5654, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278032384 dispatch_meta_bytes=228424 combine_payload_bytes=278032384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278260808 ingress_bytes=388132864 link_bytes=388132864 token_bytes=556293192 weight_movement_bytes=110100480 total_nvlink_bytes=666393672 weight_to_token_ratio=0.197918 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750268874e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280397824 dispatch_meta_bytes=229032 combine_payload_bytes=280397824 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280626856 ingress_bytes=610699264 link_bytes=610699264 token_bytes=561024680 weight_movement_bytes=330301440 total_nvlink_bytes=891326120 weight_to_token_ratio=0.588747 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062106463e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38788 dup_rows=21070 dup_rows_remote=18318 dedup_ratio=0.321503 router_tokens_per_expert=[520, 496, 537, 500, 517, 529, 488, 543, 492, 498, 496, 522, 474, 515, 538, 543, 534, 478, 476, 522, 501, 517, 502, 546, 525, 475, 556, 526, 531, 534, 514, 496, 517, 522, 499, 481, 502, 504, 508, 486, 531, 477, 543, 520, 519, 552, 526, 520, 497, 527, 503, 501, 520, 486, 547, 509, 460, 531, 509, 502, 478, 487, 510, 553] payload_rows_per_dst=[5678, 5442, 5594, 5576, 5540, 5562, 5576, 5498] dup_rows_per_dst=[2752, 2548, 2684, 2612, 2498, 2810, 2604, 2562] combine_rows_per_src=[5678, 5442, 5594, 5576, 5540, 5562, 5576, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38916 dup_rows=21010 dup_rows_remote=18338 dedup_ratio=0.320587 router_tokens_per_expert=[498, 536, 491, 514, 495, 493, 490, 531, 506, 489, 502, 525, 519, 518, 547, 507, 542, 522, 502, 541, 501, 508, 503, 528, 495, 453, 500, 509, 516, 528, 523, 502, 504, 521, 533, 548, 499, 518, 502, 516, 528, 527, 529, 515, 541, 528, 506, 517, 492, 514, 514, 468, 533, 558, 491, 511, 507, 508, 508, 481, 523, 483, 498, 513] payload_rows_per_dst=[5550, 5534, 5584, 5522, 5610, 5624, 5570, 5532] dup_rows_per_dst=[2546, 2692, 2710, 2530, 2672, 2758, 2592, 2510] combine_rows_per_src=[5550, 5534, 5584, 5522, 5610, 5624, 5570, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278175744 dispatch_meta_bytes=229832 combine_payload_bytes=278175744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278405576 ingress_bytes=388276224 link_bytes=388276224 token_bytes=556581320 weight_movement_bytes=110100480 total_nvlink_bytes=666681800 weight_to_token_ratio=0.197816 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752762140e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=230280 combine_payload_bytes=279881728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280112008 ingress_bytes=389982208 link_bytes=389982208 token_bytes=559993736 weight_movement_bytes=110100480 total_nvlink_bytes=670094216 weight_to_token_ratio=0.196610 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782432008e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278562816 dispatch_meta_bytes=229232 combine_payload_bytes=278562816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278792048 ingress_bytes=388663296 link_bytes=388663296 token_bytes=557354864 weight_movement_bytes=110100480 total_nvlink_bytes=667455344 weight_to_token_ratio=0.197541 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759493959e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280397824 dispatch_meta_bytes=229032 combine_payload_bytes=280397824 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280626856 ingress_bytes=610699264 link_bytes=610699264 token_bytes=561024680 weight_movement_bytes=330301440 total_nvlink_bytes=891326120 weight_to_token_ratio=0.588747 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062106463e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=228384 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279335968 ingress_bytes=609409024 link_bytes=609409024 token_bytes=558443552 weight_movement_bytes=330301440 total_nvlink_bytes=888744992 weight_to_token_ratio=0.591468 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059862523e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44432 payload_rows_remote=38818 dup_rows=21104 dup_rows_remote=18442 dedup_ratio=0.322021 router_tokens_per_expert=[544, 517, 501, 510, 508, 490, 544, 519, 500, 556, 529, 498, 509, 513, 504, 487, 526, 504, 499, 514, 486, 492, 510, 539, 498, 497, 522, 505, 529, 520, 525, 542, 479, 488, 481, 509, 499, 523, 576, 500, 499, 503, 519, 470, 515, 501, 547, 496, 506, 532, 562, 509, 560, 487, 506, 512, 513, 513, 505, 491, 500, 533, 514, 483] payload_rows_per_dst=[5586, 5510, 5550, 5614, 5548, 5450, 5676, 5498] dup_rows_per_dst=[2680, 2682, 2590, 2662, 2562, 2650, 2672, 2606] combine_rows_per_src=[5586, 5510, 5550, 5614, 5548, 5450, 5676, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278032384 dispatch_meta_bytes=228424 combine_payload_bytes=278032384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278260808 ingress_bytes=388132864 link_bytes=388132864 token_bytes=556293192 weight_movement_bytes=110100480 total_nvlink_bytes=666393672 weight_to_token_ratio=0.197918 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750268874e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278519808 dispatch_meta_bytes=229248 combine_payload_bytes=278519808 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278749056 ingress_bytes=608821248 link_bytes=608821248 token_bytes=557268864 weight_movement_bytes=330301440 total_nvlink_bytes=887570304 weight_to_token_ratio=0.592715 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058840284e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=230096 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279710416 ingress_bytes=389580800 link_bytes=389580800 token_bytes=559190736 weight_movement_bytes=110100480 total_nvlink_bytes=669291216 weight_to_token_ratio=0.196893 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775450862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38916 dup_rows=21010 dup_rows_remote=18338 dedup_ratio=0.320587 router_tokens_per_expert=[498, 536, 491, 514, 495, 493, 490, 531, 506, 489, 502, 525, 519, 518, 547, 507, 542, 522, 502, 541, 501, 508, 503, 528, 495, 453, 500, 509, 516, 528, 523, 502, 504, 521, 533, 548, 499, 518, 502, 516, 528, 527, 529, 515, 541, 528, 506, 517, 492, 514, 514, 468, 533, 558, 491, 511, 507, 508, 508, 481, 523, 483, 498, 513] payload_rows_per_dst=[5550, 5534, 5584, 5522, 5610, 5624, 5570, 5532] dup_rows_per_dst=[2546, 2692, 2710, 2530, 2672, 2758, 2592, 2510] combine_rows_per_src=[5550, 5534, 5584, 5522, 5610, 5624, 5570, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278247424 dispatch_meta_bytes=229040 combine_payload_bytes=278247424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278476464 ingress_bytes=388347904 link_bytes=388347904 token_bytes=556723888 weight_movement_bytes=110100480 total_nvlink_bytes=666824368 weight_to_token_ratio=0.197765 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754008773e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44542 payload_rows_remote=38830 dup_rows=20994 dup_rows_remote=18174 dedup_ratio=0.320343 router_tokens_per_expert=[513, 513, 528, 498, 525, 536, 531, 512, 525, 544, 502, 545, 483, 535, 515, 493, 500, 512, 495, 531, 522, 479, 504, 504, 539, 510, 507, 483, 515, 483, 502, 508, 498, 511, 526, 518, 527, 545, 469, 524, 507, 534, 501, 511, 496, 512, 523, 489, 542, 512, 519, 542, 502, 488, 508, 543, 486, 524, 517, 508, 516, 496, 511, 471] payload_rows_per_dst=[5712, 5636, 5540, 5556, 5548, 5510, 5482, 5558] dup_rows_per_dst=[2820, 2682, 2564, 2538, 2688, 2592, 2610, 2500] combine_rows_per_src=[5712, 5636, 5540, 5556, 5548, 5510, 5482, 5558] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=229016 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279178904 ingress_bytes=389050368 link_bytes=389050368 token_bytes=558128792 weight_movement_bytes=110100480 total_nvlink_bytes=668229272 weight_to_token_ratio=0.197267 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766225777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44202 payload_rows_remote=38614 dup_rows=21334 dup_rows_remote=18626 dedup_ratio=0.325531 router_tokens_per_expert=[507, 521, 503, 512, 508, 533, 511, 509, 546, 507, 534, 524, 522, 488, 515, 512, 544, 504, 518, 508, 518, 541, 510, 502, 508, 508, 499, 513, 487, 493, 527, 505, 480, 483, 531, 517, 517, 501, 555, 482, 496, 507, 545, 531, 511, 499, 518, 507, 528, 489, 519, 442, 530, 506, 536, 515, 509, 509, 527, 497, 503, 536, 493, 512] payload_rows_per_dst=[5522, 5588, 5554, 5478, 5492, 5578, 5468, 5522] dup_rows_per_dst=[2686, 2708, 2736, 2602, 2640, 2650, 2662, 2650] combine_rows_per_src=[5522, 5588, 5554, 5478, 5492, 5578, 5468, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44360 payload_rows_remote=38720 dup_rows=21176 dup_rows_remote=18206 dedup_ratio=0.323120 router_tokens_per_expert=[484, 485, 500, 522, 499, 505, 491, 512, 530, 459, 513, 486, 469, 500, 535, 501, 516, 510, 539, 511, 542, 483, 504, 533, 535, 547, 510, 534, 486, 541, 522, 522, 473, 546, 511, 542, 543, 493, 514, 515, 492, 522, 519, 485, 514, 524, 495, 480, 556, 495, 508, 499, 508, 521, 513, 531, 531, 490, 509, 512, 519, 546, 523, 513] payload_rows_per_dst=[5502, 5450, 5510, 5650, 5630, 5510, 5468, 5640] dup_rows_per_dst=[2494, 2458, 2634, 2774, 2746, 2552, 2548, 2970] combine_rows_per_src=[5502, 5450, 5510, 5650, 5630, 5510, 5468, 5640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44606 payload_rows_remote=38938 dup_rows=20930 dup_rows_remote=18158 dedup_ratio=0.319366 router_tokens_per_expert=[484, 522, 496, 488, 516, 525, 503, 552, 500, 515, 499, 521, 483, 505, 511, 522, 496, 501, 542, 526, 539, 528, 503, 498, 502, 494, 476, 501, 523, 516, 516, 527, 485, 483, 510, 509, 538, 472, 511, 521, 561, 513, 513, 523, 459, 532, 505, 503, 529, 541, 542, 534, 529, 506, 534, 505, 519, 539, 461, 494, 496, 504, 521, 546] payload_rows_per_dst=[5588, 5518, 5682, 5516, 5552, 5592, 5668, 5490] dup_rows_per_dst=[2584, 2594, 2584, 2594, 2506, 2626, 2772, 2670] combine_rows_per_src=[5588, 5518, 5682, 5516, 5552, 5592, 5668, 5490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278247424 dispatch_meta_bytes=229040 combine_payload_bytes=278247424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278476464 ingress_bytes=388347904 link_bytes=388347904 token_bytes=556723888 weight_movement_bytes=110100480 total_nvlink_bytes=666824368 weight_to_token_ratio=0.197765 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754008773e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39024 dup_rows=20968 dup_rows_remote=18350 dedup_ratio=0.319946 router_tokens_per_expert=[540, 474, 482, 518, 492, 510, 537, 521, 545, 504, 509, 505, 532, 518, 511, 488, 531, 510, 493, 520, 519, 523, 493, 503, 524, 519, 511, 541, 511, 530, 470, 561, 501, 533, 502, 502, 469, 540, 526, 533, 492, 511, 519, 501, 504, 508, 532, 514, 509, 513, 500, 481, 528, 487, 507, 518, 499, 484, 531, 535, 486, 514, 532, 512] payload_rows_per_dst=[5550, 5606, 5582, 5616, 5584, 5544, 5468, 5618] dup_rows_per_dst=[2598, 2618, 2602, 2718, 2628, 2618, 2618, 2568] combine_rows_per_src=[5550, 5606, 5582, 5616, 5584, 5544, 5468, 5618] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=228016 combine_payload_bytes=278333440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278561456 ingress_bytes=388433920 link_bytes=388433920 token_bytes=556894896 weight_movement_bytes=110100480 total_nvlink_bytes=666995376 weight_to_token_ratio=0.197704 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755504733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=38972 dup_rows=21006 dup_rows_remote=18450 dedup_ratio=0.320526 router_tokens_per_expert=[483, 522, 513, 498, 538, 502, 544, 459, 531, 496, 540, 498, 522, 516, 520, 507, 504, 488, 549, 510, 520, 531, 500, 455, 496, 524, 518, 499, 492, 514, 522, 518, 496, 529, 517, 541, 497, 488, 476, 521, 539, 541, 530, 489, 505, 520, 497, 520, 472, 504, 509, 532, 538, 543, 491, 542, 536, 511, 496, 500, 525, 513, 536, 485] payload_rows_per_dst=[5490, 5638, 5558, 5562, 5562, 5596, 5538, 5586] dup_rows_per_dst=[2628, 2622, 2556, 2604, 2568, 2686, 2724, 2618] combine_rows_per_src=[5490, 5638, 5558, 5562, 5562, 5596, 5538, 5586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276785152 dispatch_meta_bytes=228960 combine_payload_bytes=276785152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277014112 ingress_bytes=496986112 link_bytes=496986112 token_bytes=553799264 weight_movement_bytes=220200960 total_nvlink_bytes=774000224 weight_to_token_ratio=0.397619 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.643405889e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=228384 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279335968 ingress_bytes=609409024 link_bytes=609409024 token_bytes=558443552 weight_movement_bytes=330301440 total_nvlink_bytes=888744992 weight_to_token_ratio=0.591468 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059862523e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44202 payload_rows_remote=38614 dup_rows=21334 dup_rows_remote=18626 dedup_ratio=0.325531 router_tokens_per_expert=[507, 521, 503, 512, 508, 533, 511, 509, 546, 507, 534, 524, 522, 488, 515, 512, 544, 504, 518, 508, 518, 541, 510, 502, 508, 508, 499, 513, 487, 493, 527, 505, 480, 483, 531, 517, 517, 501, 555, 482, 496, 507, 545, 531, 511, 499, 518, 507, 528, 489, 519, 442, 530, 506, 536, 515, 509, 509, 527, 497, 503, 536, 493, 512] payload_rows_per_dst=[5522, 5588, 5554, 5478, 5492, 5578, 5468, 5522] dup_rows_per_dst=[2686, 2708, 2736, 2602, 2640, 2650, 2662, 2650] combine_rows_per_src=[5522, 5588, 5554, 5478, 5492, 5578, 5468, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38990 dup_rows=21026 dup_rows_remote=18534 dedup_ratio=0.320831 router_tokens_per_expert=[471, 510, 526, 512, 517, 510, 492, 534, 537, 507, 501, 505, 546, 559, 509, 505, 504, 483, 505, 517, 493, 496, 500, 508, 495, 520, 497, 500, 543, 495, 511, 511, 517, 488, 513, 536, 483, 516, 496, 478, 469, 528, 505, 531, 569, 527, 552, 499, 512, 528, 504, 527, 529, 491, 548, 526, 526, 512, 522, 523, 518, 480, 475, 521] payload_rows_per_dst=[5506, 5634, 5520, 5500, 5474, 5640, 5654, 5582] dup_rows_per_dst=[2638, 2704, 2492, 2644, 2580, 2720, 2676, 2572] combine_rows_per_src=[5506, 5634, 5520, 5500, 5474, 5640, 5654, 5582] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277544960 dispatch_meta_bytes=227704 combine_payload_bytes=277544960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277772664 ingress_bytes=387645440 link_bytes=387645440 token_bytes=555317624 weight_movement_bytes=110100480 total_nvlink_bytes=665418104 weight_to_token_ratio=0.198266 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.741791769e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44474 payload_rows_remote=38950 dup_rows=21062 dup_rows_remote=18470 dedup_ratio=0.321381 router_tokens_per_expert=[486, 531, 523, 518, 520, 467, 528, 533, 497, 472, 559, 528, 506, 481, 526, 491, 521, 518, 532, 528, 494, 517, 527, 505, 505, 495, 525, 525, 489, 494, 488, 494, 504, 525, 525, 505, 521, 505, 516, 508, 563, 508, 473, 501, 531, 534, 538, 533, 502, 491, 513, 525, 511, 503, 478, 535, 473, 540, 480, 525, 506, 515, 505, 553] payload_rows_per_dst=[5574, 5524, 5602, 5494, 5574, 5614, 5524, 5568] dup_rows_per_dst=[2638, 2596, 2682, 2536, 2644, 2748, 2592, 2626] combine_rows_per_src=[5574, 5524, 5602, 5494, 5574, 5614, 5524, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44542 payload_rows_remote=38830 dup_rows=20994 dup_rows_remote=18174 dedup_ratio=0.320343 router_tokens_per_expert=[513, 513, 528, 498, 525, 536, 531, 512, 525, 544, 502, 545, 483, 535, 515, 493, 500, 512, 495, 531, 522, 479, 504, 504, 539, 510, 507, 483, 515, 483, 502, 508, 498, 511, 526, 518, 527, 545, 469, 524, 507, 534, 501, 511, 496, 512, 523, 489, 542, 512, 519, 542, 502, 488, 508, 543, 486, 524, 517, 508, 516, 496, 511, 471] payload_rows_per_dst=[5712, 5636, 5540, 5556, 5548, 5510, 5482, 5558] dup_rows_per_dst=[2820, 2682, 2564, 2538, 2688, 2592, 2610, 2500] combine_rows_per_src=[5712, 5636, 5540, 5556, 5548, 5510, 5482, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39024 dup_rows=20968 dup_rows_remote=18350 dedup_ratio=0.319946 router_tokens_per_expert=[540, 474, 482, 518, 492, 510, 537, 521, 545, 504, 509, 505, 532, 518, 511, 488, 531, 510, 493, 520, 519, 523, 493, 503, 524, 519, 511, 541, 511, 530, 470, 561, 501, 533, 502, 502, 469, 540, 526, 533, 492, 511, 519, 501, 504, 508, 532, 514, 509, 513, 500, 481, 528, 487, 507, 518, 499, 484, 531, 535, 486, 514, 532, 512] payload_rows_per_dst=[5550, 5606, 5582, 5616, 5584, 5544, 5468, 5618] dup_rows_per_dst=[2598, 2618, 2602, 2718, 2628, 2618, 2618, 2568] combine_rows_per_src=[5550, 5606, 5582, 5616, 5584, 5544, 5468, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44498 payload_rows_remote=38946 dup_rows=21038 dup_rows_remote=18384 dedup_ratio=0.321014 router_tokens_per_expert=[532, 548, 514, 468, 514, 497, 512, 500, 509, 488, 532, 520, 501, 497, 497, 519, 499, 513, 490, 494, 520, 561, 502, 492, 525, 513, 509, 507, 488, 522, 530, 512, 468, 523, 540, 556, 518, 557, 473, 468, 542, 509, 545, 515, 514, 485, 525, 517, 494, 515, 529, 566, 495, 530, 514, 487, 494, 510, 480, 507, 530, 505, 538, 494] payload_rows_per_dst=[5494, 5510, 5546, 5584, 5552, 5656, 5590, 5566] dup_rows_per_dst=[2676, 2616, 2596, 2628, 2654, 2648, 2670, 2550] combine_rows_per_src=[5494, 5510, 5546, 5584, 5552, 5656, 5590, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276785152 dispatch_meta_bytes=228960 combine_payload_bytes=276785152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277014112 ingress_bytes=496986112 link_bytes=496986112 token_bytes=553799264 weight_movement_bytes=220200960 total_nvlink_bytes=774000224 weight_to_token_ratio=0.397619 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.643405889e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=230096 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279710416 ingress_bytes=389580800 link_bytes=389580800 token_bytes=559190736 weight_movement_bytes=110100480 total_nvlink_bytes=669291216 weight_to_token_ratio=0.196893 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775450862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38890 dup_rows=21014 dup_rows_remote=18388 dedup_ratio=0.320648 router_tokens_per_expert=[527, 521, 524, 508, 553, 493, 492, 559, 483, 522, 520, 555, 528, 501, 512, 508, 522, 498, 472, 484, 484, 487, 506, 508, 493, 544, 516, 515, 531, 493, 528, 509, 501, 505, 496, 504, 491, 503, 496, 530, 504, 494, 549, 515, 533, 531, 525, 535, 515, 510, 518, 498, 486, 485, 522, 535, 534, 504, 534, 509, 509, 502, 521, 478] payload_rows_per_dst=[5592, 5542, 5420, 5632, 5564, 5624, 5600, 5548] dup_rows_per_dst=[2762, 2716, 2502, 2626, 2488, 2748, 2538, 2634] combine_rows_per_src=[5592, 5542, 5420, 5632, 5564, 5624, 5600, 5548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44498 payload_rows_remote=38946 dup_rows=21038 dup_rows_remote=18384 dedup_ratio=0.321014 router_tokens_per_expert=[532, 548, 514, 468, 514, 497, 512, 500, 509, 488, 532, 520, 501, 497, 497, 519, 499, 513, 490, 494, 520, 561, 502, 492, 525, 513, 509, 507, 488, 522, 530, 512, 468, 523, 540, 556, 518, 557, 473, 468, 542, 509, 545, 515, 514, 485, 525, 517, 494, 515, 529, 566, 495, 530, 514, 487, 494, 510, 480, 507, 530, 505, 538, 494] payload_rows_per_dst=[5494, 5510, 5546, 5584, 5552, 5656, 5590, 5566] dup_rows_per_dst=[2676, 2616, 2596, 2628, 2654, 2648, 2670, 2550] combine_rows_per_src=[5494, 5510, 5546, 5584, 5552, 5656, 5590, 5566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279193600 dispatch_meta_bytes=229680 combine_payload_bytes=279193600 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279423280 ingress_bytes=609495040 link_bytes=609495040 token_bytes=558616880 weight_movement_bytes=330301440 total_nvlink_bytes=888918320 weight_to_token_ratio=0.591284 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060012119e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=228016 combine_payload_bytes=278333440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278561456 ingress_bytes=388433920 link_bytes=388433920 token_bytes=556894896 weight_movement_bytes=110100480 total_nvlink_bytes=666995376 weight_to_token_ratio=0.197704 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755504733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279724032 dispatch_meta_bytes=229496 combine_payload_bytes=279724032 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279953528 ingress_bytes=499924992 link_bytes=499924992 token_bytes=559677560 weight_movement_bytes=220200960 total_nvlink_bytes=779878520 weight_to_token_ratio=0.393443 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694517846e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=229688 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279580984 ingress_bytes=609652736 link_bytes=609652736 token_bytes=558932280 weight_movement_bytes=330301440 total_nvlink_bytes=889233720 weight_to_token_ratio=0.590951 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060286379e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229112 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278992632 ingress_bytes=498964480 link_bytes=498964480 token_bytes=557756152 weight_movement_bytes=220200960 total_nvlink_bytes=777957112 weight_to_token_ratio=0.394798 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677812963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229320 combine_payload_bytes=279164928 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279394248 ingress_bytes=499365888 link_bytes=499365888 token_bytes=558559176 weight_movement_bytes=220200960 total_nvlink_bytes=778760136 weight_to_token_ratio=0.394230 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684794108e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38856 dup_rows=21052 dup_rows_remote=18456 dedup_ratio=0.321228 router_tokens_per_expert=[484, 524, 510, 531, 508, 511, 522, 504, 494, 510, 551, 535, 475, 498, 527, 528, 533, 490, 526, 495, 525, 472, 526, 501, 492, 542, 505, 485, 468, 508, 505, 537, 484, 531, 518, 547, 547, 497, 501, 552, 521, 532, 526, 539, 513, 509, 506, 498, 517, 492, 495, 487, 513, 528, 493, 515, 522, 489, 512, 508, 523, 534, 509, 488] payload_rows_per_dst=[5628, 5646, 5520, 5484, 5674, 5444, 5522, 5566] dup_rows_per_dst=[2596, 2952, 2616, 2576, 2680, 2482, 2546, 2604] combine_rows_per_src=[5628, 5646, 5520, 5484, 5674, 5444, 5522, 5566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38982 dup_rows=21082 dup_rows_remote=18438 dedup_ratio=0.321686 router_tokens_per_expert=[510, 526, 526, 499, 524, 517, 539, 525, 501, 527, 507, 503, 493, 503, 540, 484, 542, 524, 479, 510, 498, 506, 515, 507, 507, 522, 500, 524, 513, 498, 468, 528, 523, 499, 521, 530, 477, 534, 476, 533, 484, 510, 523, 456, 500, 526, 533, 524, 534, 495, 534, 484, 517, 493, 533, 531, 520, 512, 536, 500, 512, 528, 505, 520] payload_rows_per_dst=[5650, 5472, 5610, 5502, 5600, 5506, 5600, 5514] dup_rows_per_dst=[2682, 2644, 2552, 2618, 2586, 2606, 2642, 2752] combine_rows_per_src=[5650, 5472, 5610, 5502, 5600, 5506, 5600, 5514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44474 payload_rows_remote=38950 dup_rows=21062 dup_rows_remote=18470 dedup_ratio=0.321381 router_tokens_per_expert=[486, 531, 523, 518, 520, 467, 528, 533, 497, 472, 559, 528, 506, 481, 526, 491, 521, 518, 532, 528, 494, 517, 527, 505, 505, 495, 525, 525, 489, 494, 488, 494, 504, 525, 525, 505, 521, 505, 516, 508, 563, 508, 473, 501, 531, 534, 538, 533, 502, 491, 513, 525, 511, 503, 478, 535, 473, 540, 480, 525, 506, 515, 505, 553] payload_rows_per_dst=[5574, 5524, 5602, 5494, 5574, 5614, 5524, 5568] dup_rows_per_dst=[2638, 2596, 2682, 2536, 2644, 2748, 2592, 2626] combine_rows_per_src=[5574, 5524, 5602, 5494, 5574, 5614, 5524, 5568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38840 dup_rows=21122 dup_rows_remote=18464 dedup_ratio=0.322296 router_tokens_per_expert=[498, 520, 532, 506, 531, 493, 495, 515, 505, 509, 517, 532, 500, 523, 496, 511, 471, 512, 515, 521, 512, 526, 516, 511, 506, 497, 554, 526, 568, 544, 510, 496, 476, 478, 527, 518, 511, 505, 556, 469, 536, 545, 518, 560, 485, 475, 474, 494, 492, 544, 520, 548, 500, 512, 484, 502, 479, 506, 526, 548, 480, 482, 529, 521] payload_rows_per_dst=[5442, 5504, 5624, 5712, 5444, 5560, 5554, 5574] dup_rows_per_dst=[2424, 2682, 2858, 2690, 2616, 2630, 2564, 2658] combine_rows_per_src=[5442, 5504, 5624, 5712, 5444, 5560, 5554, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5577991736 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2790141496 ingress_bytes=4659558400 link_bytes=4659558400 total_nvlink_bytes=7449699896 weight_to_token_ratio=0.335552 prefetch_share_of_ingress=0.236290 dedup_ratio=0.321082 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.103738423e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229112 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278992632 ingress_bytes=498964480 link_bytes=498964480 token_bytes=557756152 weight_movement_bytes=220200960 total_nvlink_bytes=777957112 weight_to_token_ratio=0.394798 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677812963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5591585704 prefetch_bytes=1101004800 reduce_bytes=1101004800 weight_movement_bytes=2202009600 egress_bytes=2796940200 ingress_bytes=4996655104 link_bytes=4996655104 total_nvlink_bytes=7793595304 weight_to_token_ratio=0.393808 prefetch_share_of_ingress=0.220348 dedup_ratio=0.320053 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=10 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.690005034e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582810416 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=2792551728 ingress_bytes=4331665408 link_bytes=4331665408 total_nvlink_bytes=7124217136 weight_to_token_ratio=0.276099 prefetch_share_of_ingress=0.228758 dedup_ratio=0.321255 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.533478581e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38982 dup_rows=21082 dup_rows_remote=18438 dedup_ratio=0.321686 router_tokens_per_expert=[510, 526, 526, 499, 524, 517, 539, 525, 501, 527, 507, 503, 493, 503, 540, 484, 542, 524, 479, 510, 498, 506, 515, 507, 507, 522, 500, 524, 513, 498, 468, 528, 523, 499, 521, 530, 477, 534, 476, 533, 484, 510, 523, 456, 500, 526, 533, 524, 534, 495, 534, 484, 517, 493, 533, 531, 520, 512, 536, 500, 512, 528, 505, 520] payload_rows_per_dst=[5650, 5472, 5610, 5502, 5600, 5506, 5600, 5514] dup_rows_per_dst=[2682, 2644, 2552, 2618, 2586, 2606, 2642, 2752] combine_rows_per_src=[5650, 5472, 5610, 5502, 5600, 5506, 5600, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278405120 dispatch_meta_bytes=229216 combine_payload_bytes=278405120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278634336 ingress_bytes=388505600 link_bytes=388505600 token_bytes=557039456 weight_movement_bytes=110100480 total_nvlink_bytes=667139936 weight_to_token_ratio=0.197653 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756751366e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5577991736 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2790141496 ingress_bytes=4659558400 link_bytes=4659558400 total_nvlink_bytes=7449699896 weight_to_token_ratio=0.335552 prefetch_share_of_ingress=0.236290 dedup_ratio=0.321082 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.103738423e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=38972 dup_rows=21006 dup_rows_remote=18450 dedup_ratio=0.320526 router_tokens_per_expert=[483, 522, 513, 498, 538, 502, 544, 459, 531, 496, 540, 498, 522, 516, 520, 507, 504, 488, 549, 510, 520, 531, 500, 455, 496, 524, 518, 499, 492, 514, 522, 518, 496, 529, 517, 541, 497, 488, 476, 521, 539, 541, 530, 489, 505, 520, 497, 520, 472, 504, 509, 532, 538, 543, 491, 542, 536, 511, 496, 500, 525, 513, 536, 485] payload_rows_per_dst=[5490, 5638, 5558, 5562, 5562, 5596, 5538, 5586] dup_rows_per_dst=[2628, 2622, 2556, 2604, 2568, 2686, 2724, 2618] combine_rows_per_src=[5490, 5638, 5558, 5562, 5562, 5596, 5538, 5586] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38856 dup_rows=21052 dup_rows_remote=18456 dedup_ratio=0.321228 router_tokens_per_expert=[484, 524, 510, 531, 508, 511, 522, 504, 494, 510, 551, 535, 475, 498, 527, 528, 533, 490, 526, 495, 525, 472, 526, 501, 492, 542, 505, 485, 468, 508, 505, 537, 484, 531, 518, 547, 547, 497, 501, 552, 521, 532, 526, 539, 513, 509, 506, 498, 517, 492, 495, 487, 513, 528, 493, 515, 522, 489, 512, 508, 523, 534, 509, 488] payload_rows_per_dst=[5628, 5646, 5520, 5484, 5674, 5444, 5522, 5566] dup_rows_per_dst=[2596, 2952, 2616, 2576, 2680, 2482, 2546, 2604] combine_rows_per_src=[5628, 5646, 5520, 5484, 5674, 5444, 5522, 5566] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=39022 dup_rows=21024 dup_rows_remote=18378 dedup_ratio=0.320801 router_tokens_per_expert=[525, 526, 481, 587, 512, 510, 468, 486, 443, 490, 516, 532, 501, 495, 500, 516, 539, 515, 549, 501, 504, 513, 509, 494, 527, 476, 534, 508, 521, 514, 511, 507, 481, 560, 518, 512, 527, 508, 511, 521, 512, 521, 517, 516, 493, 514, 498, 497, 520, 529, 520, 553, 512, 515, 465, 543, 496, 478, 545, 490, 500, 513, 550, 523] payload_rows_per_dst=[5598, 5456, 5572, 5606, 5622, 5490, 5652, 5516] dup_rows_per_dst=[2592, 2530, 2676, 2590, 2654, 2646, 2662, 2674] combine_rows_per_src=[5598, 5456, 5572, 5606, 5622, 5490, 5652, 5516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278519808 dispatch_meta_bytes=229248 combine_payload_bytes=278519808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278749056 ingress_bytes=498720768 link_bytes=498720768 token_bytes=557268864 weight_movement_bytes=220200960 total_nvlink_bytes=777469824 weight_to_token_ratio=0.395143 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673574410e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5585359208 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=2793824616 ingress_bytes=4553142272 link_bytes=4553142272 total_nvlink_bytes=7346966888 weight_to_token_ratio=0.315397 prefetch_share_of_ingress=0.193450 dedup_ratio=0.320291 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.918663274e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229680 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279652656 ingress_bytes=389523456 link_bytes=389523456 token_bytes=559075632 weight_movement_bytes=110100480 total_nvlink_bytes=669176112 weight_to_token_ratio=0.196933 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774453556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38812 dup_rows=21160 dup_rows_remote=18564 dedup_ratio=0.322876 router_tokens_per_expert=[526, 507, 493, 523, 509, 519, 527, 490, 501, 518, 496, 490, 506, 527, 505, 530, 518, 509, 521, 488, 500, 513, 514, 517, 485, 512, 525, 520, 527, 527, 540, 493, 512, 497, 507, 517, 555, 473, 500, 531, 468, 527, 502, 482, 509, 519, 531, 529, 475, 565, 542, 493, 508, 515, 522, 520, 526, 493, 498, 490, 526, 514, 507, 539] payload_rows_per_dst=[5502, 5488, 5564, 5646, 5570, 5542, 5552, 5512] dup_rows_per_dst=[2686, 2658, 2596, 2612, 2614, 2592, 2728, 2674] combine_rows_per_src=[5502, 5488, 5564, 5646, 5570, 5542, 5552, 5512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279193600 dispatch_meta_bytes=229680 combine_payload_bytes=279193600 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279423280 ingress_bytes=609495040 link_bytes=609495040 token_bytes=558616880 weight_movement_bytes=330301440 total_nvlink_bytes=888918320 weight_to_token_ratio=0.591284 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060012119e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278405120 dispatch_meta_bytes=229216 combine_payload_bytes=278405120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278634336 ingress_bytes=388505600 link_bytes=388505600 token_bytes=557039456 weight_movement_bytes=110100480 total_nvlink_bytes=667139936 weight_to_token_ratio=0.197653 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756751366e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5566603728 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=2784444880 ingress_bytes=4543766528 link_bytes=4543766528 total_nvlink_bytes=7328211408 weight_to_token_ratio=0.316460 prefetch_share_of_ingress=0.193849 dedup_ratio=0.321597 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.902357313e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=229688 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279580984 ingress_bytes=609652736 link_bytes=609652736 token_bytes=558932280 weight_movement_bytes=330301440 total_nvlink_bytes=889233720 weight_to_token_ratio=0.590951 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060286379e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278519808 dispatch_meta_bytes=229248 combine_payload_bytes=278519808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278749056 ingress_bytes=498720768 link_bytes=498720768 token_bytes=557268864 weight_movement_bytes=220200960 total_nvlink_bytes=777469824 weight_to_token_ratio=0.395143 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673574410e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279709696 dispatch_meta_bytes=229600 combine_payload_bytes=279709696 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279939296 ingress_bytes=499910656 link_bytes=499910656 token_bytes=559648992 weight_movement_bytes=220200960 total_nvlink_bytes=779849952 weight_to_token_ratio=0.393463 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694268520e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5581349232 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=2791821680 ingress_bytes=4551135232 link_bytes=4551135232 total_nvlink_bytes=7342956912 weight_to_token_ratio=0.315624 prefetch_share_of_ingress=0.217727 dedup_ratio=0.321478 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.915172702e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278204416 dispatch_meta_bytes=229504 combine_payload_bytes=278204416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278433920 ingress_bytes=388304896 link_bytes=388304896 token_bytes=556638336 weight_movement_bytes=110100480 total_nvlink_bytes=666738816 weight_to_token_ratio=0.197795 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753260793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5591585704 prefetch_bytes=1101004800 reduce_bytes=1101004800 weight_movement_bytes=2202009600 egress_bytes=2796940200 ingress_bytes=4996655104 link_bytes=4996655104 total_nvlink_bytes=7793595304 weight_to_token_ratio=0.393808 prefetch_share_of_ingress=0.220348 dedup_ratio=0.320053 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=10 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.690005034e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5585359208 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=2793824616 ingress_bytes=4553142272 link_bytes=4553142272 total_nvlink_bytes=7346966888 weight_to_token_ratio=0.315397 prefetch_share_of_ingress=0.193450 dedup_ratio=0.320291 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.918663274e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5566603728 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=2784444880 ingress_bytes=4543766528 link_bytes=4543766528 total_nvlink_bytes=7328211408 weight_to_token_ratio=0.316460 prefetch_share_of_ingress=0.193849 dedup_ratio=0.321597 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.902357313e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5586310480 prefetch_bytes=880803840 reduce_bytes=1101004800 weight_movement_bytes=1981808640 egress_bytes=2794302800 ingress_bytes=4773816320 link_bytes=4773816320 total_nvlink_bytes=7568119120 weight_to_token_ratio=0.354762 prefetch_share_of_ingress=0.184507 dedup_ratio=0.320859 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=10 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.302451738e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38812 dup_rows=21160 dup_rows_remote=18564 dedup_ratio=0.322876 router_tokens_per_expert=[526, 507, 493, 523, 509, 519, 527, 490, 501, 518, 496, 490, 506, 527, 505, 530, 518, 509, 521, 488, 500, 513, 514, 517, 485, 512, 525, 520, 527, 527, 540, 493, 512, 497, 507, 517, 555, 473, 500, 531, 468, 527, 502, 482, 509, 519, 531, 529, 475, 565, 542, 493, 508, 515, 522, 520, 526, 493, 498, 490, 526, 514, 507, 539] payload_rows_per_dst=[5502, 5488, 5564, 5646, 5570, 5542, 5552, 5512] dup_rows_per_dst=[2686, 2658, 2596, 2612, 2614, 2592, 2728, 2674] combine_rows_per_src=[5502, 5488, 5564, 5646, 5570, 5542, 5552, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278204416 dispatch_meta_bytes=229504 combine_payload_bytes=278204416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278433920 ingress_bytes=388304896 link_bytes=388304896 token_bytes=556638336 weight_movement_bytes=110100480 total_nvlink_bytes=666738816 weight_to_token_ratio=0.197795 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753260793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5572718032 prefetch_bytes=880803840 reduce_bytes=1651507200 weight_movement_bytes=2532311040 egress_bytes=2787505616 ingress_bytes=5317523456 link_bytes=5317523456 total_nvlink_bytes=8105029072 weight_to_token_ratio=0.454412 prefetch_share_of_ingress=0.165642 dedup_ratio=0.321774 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=15 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.248047872e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none - [2026-08-26 16:58:53] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 372.0 | throughput per GPU (TFLOP/s/GPU): 331.9 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.253320E+00 | loss scale: 1.0 | grad norm: 5.080 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44520 payload_rows_remote=38982 dup_rows=21016 dup_rows_remote=18472 dedup_ratio=0.320679 router_tokens_per_expert=[509, 523, 502, 492, 489, 540, 507, 526, 479, 501, 507, 493, 492, 530, 497, 542, 493, 507, 535, 521, 550, 499, 489, 449, 530, 527, 533, 534, 499, 493, 542, 514, 505, 508, 492, 540, 514, 524, 522, 543, 503, 527, 525, 517, 526, 487, 504, 487, 518, 502, 527, 505, 499, 501, 485, 557, 527, 507, 537, 501, 517, 485, 516, 516] payload_rows_per_dst=[5584, 5538, 5478, 5636, 5580, 5538, 5596, 5570] dup_rows_per_dst=[2592, 2544, 2608, 2708, 2716, 2614, 2592, 2642] combine_rows_per_src=[5584, 5538, 5478, 5636, 5580, 5538, 5596, 5570] + [2026-08-26 18:10:58] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 367.9 | throughput per GPU (TFLOP/s/GPU): 335.6 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.248667E+00 | loss scale: 1.0 | grad norm: 5.078 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44474 payload_rows_remote=38820 dup_rows=21062 dup_rows_remote=18380 dedup_ratio=0.321381 router_tokens_per_expert=[493, 520, 522, 527, 499, 528, 508, 509, 509, 541, 528, 537, 488, 522, 524, 519, 529, 538, 498, 494, 487, 548, 485, 503, 532, 515, 486, 537, 543, 549, 518, 498, 495, 528, 495, 495, 496, 503, 446, 546, 521, 504, 503, 518, 490, 471, 528, 496, 518, 522, 531, 518, 535, 515, 483, 546, 520, 493, 527, 497, 501, 514, 486, 493] payload_rows_per_dst=[5548, 5642, 5540, 5630, 5486, 5466, 5654, 5508] dup_rows_per_dst=[2664, 2694, 2624, 2726, 2522, 2596, 2682, 2554] combine_rows_per_src=[5548, 5642, 5540, 5630, 5486, 5466, 5654, 5508] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=39084 dup_rows=20934 dup_rows_remote=18324 dedup_ratio=0.319427 router_tokens_per_expert=[496, 512, 519, 489, 535, 524, 495, 509, 534, 492, 511, 531, 533, 513, 509, 497, 520, 556, 505, 524, 526, 537, 487, 507, 514, 494, 532, 493, 502, 508, 510, 500, 518, 487, 492, 528, 494, 546, 506, 493, 522, 520, 515, 512, 519, 537, 499, 526, 518, 482, 520, 495, 484, 533, 506, 507, 539, 480, 528, 530, 524, 498, 487, 509] payload_rows_per_dst=[5576, 5624, 5666, 5534, 5518, 5638, 5518, 5528] dup_rows_per_dst=[2582, 2616, 2658, 2572, 2610, 2662, 2572, 2662] combine_rows_per_src=[5576, 5624, 5666, 5534, 5518, 5638, 5518, 5528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=39048 dup_rows=21006 dup_rows_remote=18466 dedup_ratio=0.320526 router_tokens_per_expert=[537, 496, 469, 469, 492, 481, 508, 546, 465, 514, 493, 539, 489, 486, 529, 482, 516, 502, 524, 532, 482, 498, 513, 501, 526, 519, 530, 546, 531, 519, 535, 528, 488, 521, 495, 488, 509, 538, 523, 550, 509, 491, 489, 522, 524, 504, 489, 483, 515, 531, 487, 565, 523, 515, 540, 520, 495, 562, 531, 531, 497, 471, 540, 525] payload_rows_per_dst=[5504, 5454, 5542, 5694, 5580, 5482, 5622, 5652] dup_rows_per_dst=[2492, 2540, 2594, 2774, 2644, 2540, 2770, 2652] combine_rows_per_src=[5504, 5454, 5542, 5694, 5580, 5482, 5622, 5652] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38934 dup_rows=21058 dup_rows_remote=18336 dedup_ratio=0.321320 router_tokens_per_expert=[537, 513, 456, 468, 529, 510, 458, 493, 487, 520, 551, 515, 478, 519, 518, 500, 519, 502, 500, 511, 507, 499, 484, 507, 482, 522, 522, 527, 525, 543, 489, 456, 518, 553, 508, 515, 525, 520, 532, 515, 544, 460, 544, 534, 544, 491, 491, 520, 509, 513, 517, 526, 513, 532, 561, 505, 514, 530, 540, 477, 511, 527, 515, 517] payload_rows_per_dst=[5414, 5586, 5446, 5562, 5664, 5592, 5670, 5544] dup_rows_per_dst=[2514, 2590, 2612, 2570, 2704, 2664, 2682, 2722] combine_rows_per_src=[5414, 5586, 5446, 5562, 5664, 5592, 5670, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=38960 dup_rows=21060 dup_rows_remote=18372 dedup_ratio=0.321350 router_tokens_per_expert=[519, 523, 506, 517, 513, 521, 501, 536, 504, 521, 508, 487, 569, 487, 472, 491, 515, 482, 509, 518, 489, 526, 506, 518, 546, 476, 538, 477, 494, 485, 556, 530, 526, 505, 497, 522, 504, 521, 513, 511, 550, 474, 526, 467, 485, 528, 508, 527, 560, 511, 505, 492, 510, 501, 528, 532, 517, 503, 515, 532, 528, 513, 518, 499] payload_rows_per_dst=[5626, 5474, 5516, 5516, 5568, 5572, 5620, 5584] dup_rows_per_dst=[2646, 2604, 2610, 2688, 2630, 2558, 2658, 2666] combine_rows_per_src=[5626, 5474, 5516, 5516, 5568, 5572, 5620, 5584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44364 payload_rows_remote=38876 dup_rows=21172 dup_rows_remote=18576 dedup_ratio=0.323059 router_tokens_per_expert=[518, 485, 524, 544, 549, 521, 492, 500, 530, 496, 516, 519, 476, 480, 500, 510, 483, 508, 515, 543, 505, 503, 490, 495, 482, 533, 508, 507, 532, 531, 526, 507, 498, 500, 502, 538, 515, 533, 522, 553, 494, 513, 539, 549, 499, 509, 512, 522, 512, 497, 526, 467, 503, 529, 484, 513, 532, 468, 498, 511, 517, 508, 519, 558] payload_rows_per_dst=[5562, 5492, 5488, 5616, 5558, 5540, 5550, 5558] dup_rows_per_dst=[2704, 2562, 2596, 2636, 2764, 2734, 2512, 2664] combine_rows_per_src=[5562, 5492, 5488, 5616, 5558, 5540, 5550, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44520 payload_rows_remote=38982 dup_rows=21016 dup_rows_remote=18472 dedup_ratio=0.320679 router_tokens_per_expert=[509, 523, 502, 492, 489, 540, 507, 526, 479, 501, 507, 493, 492, 530, 497, 542, 493, 507, 535, 521, 550, 499, 489, 449, 530, 527, 533, 534, 499, 493, 542, 514, 505, 508, 492, 540, 514, 524, 522, 543, 503, 527, 525, 517, 526, 487, 504, 487, 518, 502, 527, 505, 499, 501, 485, 557, 527, 507, 537, 501, 517, 485, 516, 516] payload_rows_per_dst=[5584, 5538, 5478, 5636, 5580, 5538, 5596, 5570] dup_rows_per_dst=[2592, 2544, 2608, 2708, 2716, 2614, 2592, 2642] combine_rows_per_src=[5584, 5538, 5478, 5636, 5580, 5538, 5596, 5570] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44530 payload_rows_remote=39048 dup_rows=21006 dup_rows_remote=18466 dedup_ratio=0.320526 router_tokens_per_expert=[537, 496, 469, 469, 492, 481, 508, 546, 465, 514, 493, 539, 489, 486, 529, 482, 516, 502, 524, 532, 482, 498, 513, 501, 526, 519, 530, 546, 531, 519, 535, 528, 488, 521, 495, 488, 509, 538, 523, 550, 509, 491, 489, 522, 524, 504, 489, 483, 515, 531, 487, 565, 523, 515, 540, 520, 495, 562, 531, 531, 497, 471, 540, 525] payload_rows_per_dst=[5504, 5454, 5542, 5694, 5580, 5482, 5622, 5652] dup_rows_per_dst=[2492, 2540, 2594, 2774, 2644, 2540, 2770, 2652] combine_rows_per_src=[5504, 5454, 5542, 5694, 5580, 5482, 5622, 5652] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44476 payload_rows_remote=38960 dup_rows=21060 dup_rows_remote=18372 dedup_ratio=0.321350 router_tokens_per_expert=[519, 523, 506, 517, 513, 521, 501, 536, 504, 521, 508, 487, 569, 487, 472, 491, 515, 482, 509, 518, 489, 526, 506, 518, 546, 476, 538, 477, 494, 485, 556, 530, 526, 505, 497, 522, 504, 521, 513, 511, 550, 474, 526, 467, 485, 528, 508, 527, 560, 511, 505, 492, 510, 501, 528, 532, 517, 503, 515, 532, 528, 513, 518, 499] payload_rows_per_dst=[5626, 5474, 5516, 5516, 5568, 5572, 5620, 5584] dup_rows_per_dst=[2646, 2604, 2610, 2688, 2630, 2558, 2658, 2666] combine_rows_per_src=[5626, 5474, 5516, 5516, 5568, 5572, 5620, 5584] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229816 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279652792 ingress_bytes=389523456 link_bytes=389523456 token_bytes=559075768 weight_movement_bytes=110100480 total_nvlink_bytes=669176248 weight_to_token_ratio=0.196933 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774453556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38794 dup_rows=21026 dup_rows_remote=18214 dedup_ratio=0.320831 router_tokens_per_expert=[544, 505, 507, 525, 518, 523, 516, 478, 495, 478, 520, 502, 532, 518, 528, 498, 514, 559, 523, 513, 541, 544, 471, 508, 540, 466, 481, 493, 484, 491, 505, 513, 495, 532, 493, 529, 478, 525, 534, 528, 524, 522, 517, 533, 524, 529, 478, 480, 536, 515, 540, 494, 481, 505, 499, 509, 533, 518, 480, 544, 552, 528, 476, 504] payload_rows_per_dst=[5716, 5714, 5706, 5380, 5494, 5600, 5410, 5490] dup_rows_per_dst=[2812, 2862, 2852, 2294, 2498, 2740, 2428, 2540] combine_rows_per_src=[5716, 5714, 5706, 5380, 5494, 5600, 5410, 5490] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=228800 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278490560 ingress_bytes=498462720 link_bytes=498462720 token_bytes=556752320 weight_movement_bytes=220200960 total_nvlink_bytes=776953280 weight_to_token_ratio=0.395510 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669086531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280154112 dispatch_meta_bytes=229632 combine_payload_bytes=280154112 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280383744 ingress_bytes=610455552 link_bytes=610455552 token_bytes=560537856 weight_movement_bytes=330301440 total_nvlink_bytes=890839296 weight_to_token_ratio=0.589258 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061682608e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38794 dup_rows=21026 dup_rows_remote=18214 dedup_ratio=0.320831 router_tokens_per_expert=[544, 505, 507, 525, 518, 523, 516, 478, 495, 478, 520, 502, 532, 518, 528, 498, 514, 559, 523, 513, 541, 544, 471, 508, 540, 466, 481, 493, 484, 491, 505, 513, 495, 532, 493, 529, 478, 525, 534, 528, 524, 522, 517, 533, 524, 529, 478, 480, 536, 515, 540, 494, 481, 505, 499, 509, 533, 518, 480, 544, 552, 528, 476, 504] payload_rows_per_dst=[5716, 5714, 5706, 5380, 5494, 5600, 5410, 5490] dup_rows_per_dst=[2812, 2862, 2852, 2294, 2498, 2740, 2428, 2540] combine_rows_per_src=[5716, 5714, 5706, 5380, 5494, 5600, 5410, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279078912 dispatch_meta_bytes=229080 combine_payload_bytes=279078912 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279307992 ingress_bytes=609380352 link_bytes=609380352 token_bytes=558386904 weight_movement_bytes=330301440 total_nvlink_bytes=888688344 weight_to_token_ratio=0.591528 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059812658e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229816 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279652792 ingress_bytes=389523456 link_bytes=389523456 token_bytes=559075768 weight_movement_bytes=110100480 total_nvlink_bytes=669176248 weight_to_token_ratio=0.196933 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774453556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279896064 dispatch_meta_bytes=230056 combine_payload_bytes=279896064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280126120 ingress_bytes=389996544 link_bytes=389996544 token_bytes=560022184 weight_movement_bytes=110100480 total_nvlink_bytes=670122664 weight_to_token_ratio=0.196600 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782681334e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=229328 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279494608 ingress_bytes=499466240 link_bytes=499466240 token_bytes=558759888 weight_movement_bytes=220200960 total_nvlink_bytes=778960848 weight_to_token_ratio=0.394089 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686539394e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278663168 dispatch_meta_bytes=229808 combine_payload_bytes=278663168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278892976 ingress_bytes=388763648 link_bytes=388763648 token_bytes=557556144 weight_movement_bytes=110100480 total_nvlink_bytes=667656624 weight_to_token_ratio=0.197470 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761239245e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278075392 dispatch_meta_bytes=228032 combine_payload_bytes=278075392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278303424 ingress_bytes=388175872 link_bytes=388175872 token_bytes=556378816 weight_movement_bytes=110100480 total_nvlink_bytes=666479296 weight_to_token_ratio=0.197888 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751016854e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44498 payload_rows_remote=38982 dup_rows=21038 dup_rows_remote=18492 dedup_ratio=0.321014 router_tokens_per_expert=[496, 503, 542, 498, 525, 480, 537, 502, 482, 466, 545, 513, 520, 505, 489, 511, 520, 553, 524, 495, 541, 506, 474, 537, 494, 462, 505, 511, 501, 539, 517, 522, 519, 549, 500, 530, 486, 502, 489, 498, 490, 510, 520, 509, 507, 517, 514, 542, 531, 510, 522, 510, 530, 492, 501, 551, 532, 538, 519, 488, 507, 542, 510, 488] payload_rows_per_dst=[5540, 5516, 5626, 5486, 5552, 5584, 5604, 5590] dup_rows_per_dst=[2626, 2546, 2674, 2616, 2594, 2634, 2690, 2658] combine_rows_per_src=[5540, 5516, 5626, 5486, 5552, 5584, 5604, 5590] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38926 dup_rows=21072 dup_rows_remote=18372 dedup_ratio=0.321533 router_tokens_per_expert=[538, 480, 513, 508, 486, 509, 525, 494, 511, 525, 529, 516, 493, 507, 511, 532, 490, 523, 491, 508, 521, 530, 531, 529, 540, 534, 519, 501, 492, 551, 475, 495, 510, 529, 506, 515, 517, 517, 442, 552, 523, 486, 486, 503, 491, 480, 503, 557, 518, 515, 518, 513, 526, 510, 516, 503, 494, 549, 501, 505, 530, 506, 488, 552] payload_rows_per_dst=[5544, 5488, 5590, 5602, 5592, 5506, 5538, 5604] dup_rows_per_dst=[2562, 2760, 2656, 2612, 2584, 2552, 2700, 2646] combine_rows_per_src=[5544, 5488, 5590, 5602, 5592, 5506, 5538, 5604] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38818 dup_rows=21138 dup_rows_remote=18484 dedup_ratio=0.322540 router_tokens_per_expert=[514, 516, 489, 551, 514, 485, 498, 528, 513, 509, 513, 483, 545, 527, 500, 501, 485, 528, 535, 498, 558, 476, 539, 481, 514, 504, 485, 519, 524, 523, 524, 506, 509, 542, 468, 491, 526, 542, 551, 488, 529, 473, 502, 514, 514, 545, 501, 497, 531, 469, 507, 513, 526, 554, 554, 515, 477, 517, 512, 517, 507, 498, 510, 484] payload_rows_per_dst=[5532, 5546, 5600, 5572, 5580, 5548, 5572, 5448] dup_rows_per_dst=[2658, 2636, 2600, 2626, 2654, 2602, 2766, 2596] combine_rows_per_src=[5532, 5546, 5600, 5572, 5580, 5548, 5572, 5448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44482 payload_rows_remote=38890 dup_rows=21054 dup_rows_remote=18478 dedup_ratio=0.321259 router_tokens_per_expert=[529, 484, 498, 512, 540, 493, 534, 500, 483, 496, 552, 494, 518, 538, 495, 501, 514, 523, 484, 496, 514, 540, 511, 506, 507, 513, 549, 517, 502, 486, 503, 513, 507, 507, 517, 552, 536, 502, 515, 514, 518, 502, 512, 516, 505, 489, 542, 508, 510, 485, 521, 515, 562, 513, 532, 537, 475, 467, 506, 499, 509, 520, 512, 518] payload_rows_per_dst=[5550, 5540, 5472, 5638, 5518, 5586, 5586, 5592] dup_rows_per_dst=[2630, 2614, 2454, 2728, 2630, 2778, 2644, 2576] combine_rows_per_src=[5550, 5540, 5472, 5638, 5518, 5586, 5586, 5592] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44620 payload_rows_remote=39066 dup_rows=20916 dup_rows_remote=18368 dedup_ratio=0.319153 router_tokens_per_expert=[514, 507, 506, 501, 564, 520, 554, 526, 536, 495, 483, 536, 535, 507, 508, 548, 485, 537, 462, 511, 504, 532, 490, 534, 534, 527, 456, 509, 522, 464, 512, 527, 506, 488, 506, 517, 527, 506, 510, 494, 511, 514, 528, 505, 520, 504, 546, 533, 468, 501, 527, 516, 506, 492, 512, 505, 492, 515, 510, 503, 542, 524, 515, 479] payload_rows_per_dst=[5664, 5624, 5568, 5554, 5502, 5612, 5474, 5622] dup_rows_per_dst=[2720, 2672, 2542, 2548, 2606, 2710, 2580, 2538] combine_rows_per_src=[5664, 5624, 5568, 5554, 5502, 5612, 5474, 5622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44826 payload_rows_remote=39232 dup_rows=20710 dup_rows_remote=18178 dedup_ratio=0.316010 router_tokens_per_expert=[474, 521, 523, 521, 534, 509, 516, 521, 493, 530, 484, 500, 520, 507, 496, 515, 514, 523, 519, 546, 492, 488, 487, 536, 515, 507, 501, 478, 519, 516, 491, 529, 518, 514, 512, 500, 496, 533, 560, 486, 483, 514, 514, 486, 496, 550, 512, 508, 503, 509, 502, 525, 528, 497, 499, 508, 523, 509, 536, 498, 529, 511, 528, 556] payload_rows_per_dst=[5598, 5528, 5578, 5546, 5702, 5594, 5554, 5726] dup_rows_per_dst=[2640, 2562, 2632, 2566, 2536, 2532, 2588, 2654] combine_rows_per_src=[5598, 5528, 5578, 5546, 5702, 5594, 5554, 5726] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229896 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279652872 ingress_bytes=389523456 link_bytes=389523456 token_bytes=559075848 weight_movement_bytes=110100480 total_nvlink_bytes=669176328 weight_to_token_ratio=0.196933 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774453556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44498 payload_rows_remote=38982 dup_rows=21038 dup_rows_remote=18492 dedup_ratio=0.321014 router_tokens_per_expert=[496, 503, 542, 498, 525, 480, 537, 502, 482, 466, 545, 513, 520, 505, 489, 511, 520, 553, 524, 495, 541, 506, 474, 537, 494, 462, 505, 511, 501, 539, 517, 522, 519, 549, 500, 530, 486, 502, 489, 498, 490, 510, 520, 509, 507, 517, 514, 542, 531, 510, 522, 510, 530, 492, 501, 551, 532, 538, 519, 488, 507, 542, 510, 488] payload_rows_per_dst=[5540, 5516, 5626, 5486, 5552, 5584, 5604, 5590] dup_rows_per_dst=[2626, 2546, 2674, 2616, 2594, 2634, 2690, 2658] combine_rows_per_src=[5540, 5516, 5626, 5486, 5552, 5584, 5604, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44620 payload_rows_remote=39066 dup_rows=20916 dup_rows_remote=18368 dedup_ratio=0.319153 router_tokens_per_expert=[514, 507, 506, 501, 564, 520, 554, 526, 536, 495, 483, 536, 535, 507, 508, 548, 485, 537, 462, 511, 504, 532, 490, 534, 534, 527, 456, 509, 522, 464, 512, 527, 506, 488, 506, 517, 527, 506, 510, 494, 511, 514, 528, 505, 520, 504, 546, 533, 468, 501, 527, 516, 506, 492, 512, 505, 492, 515, 510, 503, 542, 524, 515, 479] payload_rows_per_dst=[5664, 5624, 5568, 5554, 5502, 5612, 5474, 5622] dup_rows_per_dst=[2720, 2672, 2542, 2548, 2606, 2710, 2580, 2538] combine_rows_per_src=[5664, 5624, 5568, 5554, 5502, 5612, 5474, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44826 payload_rows_remote=39232 dup_rows=20710 dup_rows_remote=18178 dedup_ratio=0.316010 router_tokens_per_expert=[474, 521, 523, 521, 534, 509, 516, 521, 493, 530, 484, 500, 520, 507, 496, 515, 514, 523, 519, 546, 492, 488, 487, 536, 515, 507, 501, 478, 519, 516, 491, 529, 518, 514, 512, 500, 496, 533, 560, 486, 483, 514, 514, 486, 496, 550, 512, 508, 503, 509, 502, 525, 528, 497, 499, 508, 523, 509, 536, 498, 529, 511, 528, 556] payload_rows_per_dst=[5598, 5528, 5578, 5546, 5702, 5594, 5554, 5726] dup_rows_per_dst=[2640, 2562, 2632, 2566, 2536, 2532, 2588, 2654] combine_rows_per_src=[5598, 5528, 5578, 5546, 5702, 5594, 5554, 5726] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279021568 dispatch_meta_bytes=229192 combine_payload_bytes=279021568 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279250760 ingress_bytes=609323008 link_bytes=609323008 token_bytes=558272328 weight_movement_bytes=330301440 total_nvlink_bytes=888573768 weight_to_token_ratio=0.591649 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059712927e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38924 dup_rows=21026 dup_rows_remote=18420 dedup_ratio=0.320831 router_tokens_per_expert=[518, 512, 470, 537, 526, 527, 504, 487, 490, 515, 480, 496, 519, 491, 517, 516, 503, 533, 523, 515, 507, 515, 512, 488, 505, 532, 494, 506, 529, 502, 469, 522, 479, 540, 535, 509, 505, 502, 577, 577, 520, 529, 504, 545, 535, 474, 531, 498, 539, 474, 515, 470, 497, 513, 525, 524, 539, 532, 535, 488, 524, 479, 523, 471] payload_rows_per_dst=[5578, 5494, 5586, 5564, 5632, 5608, 5494, 5554] dup_rows_per_dst=[2584, 2554, 2606, 2554, 2816, 2664, 2620, 2628] combine_rows_per_src=[5578, 5494, 5586, 5564, 5632, 5608, 5494, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229472 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278992992 ingress_bytes=388864000 link_bytes=388864000 token_bytes=557756512 weight_movement_bytes=110100480 total_nvlink_bytes=667856992 weight_to_token_ratio=0.197399 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762984531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278247424 dispatch_meta_bytes=229208 combine_payload_bytes=278247424 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278476632 ingress_bytes=608548864 link_bytes=608548864 token_bytes=556724056 weight_movement_bytes=330301440 total_nvlink_bytes=887025496 weight_to_token_ratio=0.593295 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058366564e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44584 payload_rows_remote=39008 dup_rows=20952 dup_rows_remote=18418 dedup_ratio=0.319702 router_tokens_per_expert=[522, 513, 526, 496, 476, 515, 476, 484, 527, 517, 516, 490, 502, 533, 513, 525, 499, 514, 524, 537, 564, 488, 536, 514, 516, 544, 509, 550, 502, 520, 504, 517, 506, 492, 501, 497, 534, 528, 489, 519, 518, 473, 536, 516, 503, 475, 489, 503, 523, 553, 506, 545, 512, 511, 503, 515, 505, 526, 510, 490, 511, 514, 492, 504] payload_rows_per_dst=[5576, 5698, 5670, 5614, 5498, 5424, 5584, 5520] dup_rows_per_dst=[2534, 2758, 2682, 2710, 2540, 2602, 2542, 2584] combine_rows_per_src=[5576, 5698, 5670, 5614, 5498, 5424, 5584, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229472 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278992992 ingress_bytes=388864000 link_bytes=388864000 token_bytes=557756512 weight_movement_bytes=110100480 total_nvlink_bytes=667856992 weight_to_token_ratio=0.197399 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762984531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44364 payload_rows_remote=38876 dup_rows=21172 dup_rows_remote=18576 dedup_ratio=0.323059 router_tokens_per_expert=[518, 485, 524, 544, 549, 521, 492, 500, 530, 496, 516, 519, 476, 480, 500, 510, 483, 508, 515, 543, 505, 503, 490, 495, 482, 533, 508, 507, 532, 531, 526, 507, 498, 500, 502, 538, 515, 533, 522, 553, 494, 513, 539, 549, 499, 509, 512, 522, 512, 497, 526, 467, 503, 529, 484, 513, 532, 468, 498, 511, 517, 508, 519, 558] payload_rows_per_dst=[5562, 5492, 5488, 5616, 5558, 5540, 5550, 5558] dup_rows_per_dst=[2704, 2562, 2596, 2636, 2764, 2734, 2512, 2664] combine_rows_per_src=[5562, 5492, 5488, 5616, 5558, 5540, 5550, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229896 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279652872 ingress_bytes=389523456 link_bytes=389523456 token_bytes=559075848 weight_movement_bytes=110100480 total_nvlink_bytes=669176328 weight_to_token_ratio=0.196933 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774453556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=229736 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280254824 ingress_bytes=390125568 link_bytes=390125568 token_bytes=560279912 weight_movement_bytes=110100480 total_nvlink_bytes=670380392 weight_to_token_ratio=0.196510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.784925274e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44584 payload_rows_remote=39008 dup_rows=20952 dup_rows_remote=18418 dedup_ratio=0.319702 router_tokens_per_expert=[522, 513, 526, 496, 476, 515, 476, 484, 527, 517, 516, 490, 502, 533, 513, 525, 499, 514, 524, 537, 564, 488, 536, 514, 516, 544, 509, 550, 502, 520, 504, 517, 506, 492, 501, 497, 534, 528, 489, 519, 518, 473, 536, 516, 503, 475, 489, 503, 523, 553, 506, 545, 512, 511, 503, 515, 505, 526, 510, 490, 511, 514, 492, 504] payload_rows_per_dst=[5576, 5698, 5670, 5614, 5498, 5424, 5584, 5520] dup_rows_per_dst=[2534, 2758, 2682, 2710, 2540, 2602, 2542, 2584] combine_rows_per_src=[5576, 5698, 5670, 5614, 5498, 5424, 5584, 5520] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=281214976 dispatch_meta_bytes=229640 combine_payload_bytes=281214976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281444616 ingress_bytes=391315456 link_bytes=391315456 token_bytes=562659592 weight_movement_bytes=110100480 total_nvlink_bytes=672760072 weight_to_token_ratio=0.195679 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.805619383e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229376 combine_payload_bytes=279007232 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279236608 ingress_bytes=609308672 link_bytes=609308672 token_bytes=558243840 weight_movement_bytes=330301440 total_nvlink_bytes=888545280 weight_to_token_ratio=0.591680 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059687995e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38908 dup_rows=21002 dup_rows_remote=18324 dedup_ratio=0.320465 router_tokens_per_expert=[557, 521, 530, 511, 525, 495, 540, 467, 506, 525, 537, 501, 529, 535, 498, 521, 475, 531, 525, 508, 509, 560, 520, 494, 460, 536, 531, 529, 505, 494, 501, 532, 526, 527, 500, 526, 564, 509, 470, 479, 529, 505, 489, 473, 497, 493, 481, 543, 495, 521, 527, 513, 501, 489, 498, 533, 532, 534, 471, 528, 473, 492, 552, 490] payload_rows_per_dst=[5582, 5626, 5574, 5506, 5566, 5484, 5568, 5628] dup_rows_per_dst=[2710, 2678, 2670, 2670, 2636, 2536, 2586, 2516] combine_rows_per_src=[5582, 5626, 5574, 5506, 5566, 5484, 5568, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279609344 dispatch_meta_bytes=229704 combine_payload_bytes=279609344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279839048 ingress_bytes=389709824 link_bytes=389709824 token_bytes=559448392 weight_movement_bytes=110100480 total_nvlink_bytes=669548872 weight_to_token_ratio=0.196802 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777694802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38546 dup_rows=21170 dup_rows_remote=18310 dedup_ratio=0.323029 router_tokens_per_expert=[517, 534, 498, 499, 522, 504, 541, 505, 505, 513, 504, 491, 512, 524, 492, 519, 490, 526, 503, 502, 539, 496, 527, 538, 515, 497, 500, 542, 500, 472, 547, 516, 502, 551, 499, 509, 519, 508, 525, 522, 481, 511, 479, 492, 494, 473, 512, 490, 555, 512, 518, 556, 462, 510, 534, 513, 522, 530, 538, 525, 509, 514, 506, 507] payload_rows_per_dst=[5324, 5488, 5634, 5418, 5680, 5402, 5600, 5820] dup_rows_per_dst=[2452, 2632, 2608, 2504, 2932, 2462, 2720, 2860] combine_rows_per_src=[5324, 5488, 5634, 5418, 5680, 5402, 5600, 5820] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44342 payload_rows_remote=38772 dup_rows=21194 dup_rows_remote=18484 dedup_ratio=0.323395 router_tokens_per_expert=[487, 521, 520, 528, 515, 531, 486, 514, 502, 489, 521, 475, 543, 501, 500, 528, 526, 524, 495, 515, 521, 507, 530, 469, 489, 477, 485, 515, 550, 551, 514, 542, 481, 492, 514, 498, 522, 520, 534, 499, 519, 531, 475, 506, 490, 527, 490, 532, 490, 509, 512, 523, 507, 516, 566, 517, 459, 538, 531, 511, 510, 541, 528, 509] payload_rows_per_dst=[5546, 5526, 5556, 5538, 5532, 5524, 5570, 5550] dup_rows_per_dst=[2658, 2592, 2618, 2708, 2588, 2616, 2710, 2704] combine_rows_per_src=[5546, 5526, 5556, 5538, 5532, 5524, 5570, 5550] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44438 payload_rows_remote=38854 dup_rows=21098 dup_rows_remote=18460 dedup_ratio=0.321930 router_tokens_per_expert=[538, 477, 517, 484, 528, 504, 522, 534, 501, 507, 523, 522, 512, 534, 459, 496, 500, 497, 533, 524, 485, 525, 486, 529, 537, 461, 502, 575, 538, 535, 527, 504, 556, 497, 542, 499, 494, 495, 526, 502, 494, 476, 533, 500, 528, 486, 511, 502, 521, 506, 495, 502, 496, 514, 506, 493, 538, 516, 535, 483, 526, 551, 545, 484] payload_rows_per_dst=[5586, 5552, 5496, 5596, 5584, 5528, 5494, 5602] dup_rows_per_dst=[2622, 2556, 2662, 2762, 2638, 2532, 2572, 2754] combine_rows_per_src=[5586, 5552, 5496, 5596, 5584, 5528, 5494, 5602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38546 dup_rows=21170 dup_rows_remote=18310 dedup_ratio=0.323029 router_tokens_per_expert=[517, 534, 498, 499, 522, 504, 541, 505, 505, 513, 504, 491, 512, 524, 492, 519, 490, 526, 503, 502, 539, 496, 527, 538, 515, 497, 500, 542, 500, 472, 547, 516, 502, 551, 499, 509, 519, 508, 525, 522, 481, 511, 479, 492, 494, 473, 512, 490, 555, 512, 518, 556, 462, 510, 534, 513, 522, 530, 538, 525, 509, 514, 506, 507] payload_rows_per_dst=[5324, 5488, 5634, 5418, 5680, 5402, 5600, 5820] dup_rows_per_dst=[2452, 2632, 2608, 2504, 2932, 2462, 2720, 2860] combine_rows_per_src=[5324, 5488, 5634, 5418, 5680, 5402, 5600, 5820] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44470 payload_rows_remote=38998 dup_rows=21066 dup_rows_remote=18450 dedup_ratio=0.321442 router_tokens_per_expert=[561, 516, 497, 495, 536, 505, 504, 517, 506, 488, 540, 509, 522, 526, 523, 493, 503, 513, 509, 521, 518, 473, 555, 487, 500, 512, 509, 504, 530, 473, 509, 507, 493, 505, 517, 460, 544, 525, 503, 506, 537, 513, 531, 527, 495, 513, 497, 515, 542, 486, 491, 536, 517, 478, 525, 509, 486, 506, 525, 526, 473, 551, 525, 550] payload_rows_per_dst=[5580, 5592, 5540, 5472, 5560, 5618, 5568, 5540] dup_rows_per_dst=[2682, 2622, 2618, 2616, 2546, 2638, 2600, 2744] combine_rows_per_src=[5580, 5592, 5540, 5472, 5560, 5618, 5568, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44352 payload_rows_remote=38860 dup_rows=21184 dup_rows_remote=18610 dedup_ratio=0.323242 router_tokens_per_expert=[518, 526, 515, 523, 517, 497, 523, 513, 511, 529, 507, 530, 496, 504, 519, 504, 487, 509, 501, 501, 541, 515, 508, 515, 501, 514, 541, 538, 517, 516, 539, 512, 501, 521, 473, 514, 472, 516, 506, 538, 529, 508, 496, 494, 489, 539, 452, 526, 509, 502, 478, 511, 543, 513, 532, 495, 525, 501, 505, 498, 521, 553, 520, 501] payload_rows_per_dst=[5562, 5632, 5498, 5554, 5510, 5492, 5592, 5512] dup_rows_per_dst=[2702, 2568, 2656, 2802, 2572, 2574, 2574, 2736] combine_rows_per_src=[5562, 5632, 5498, 5554, 5510, 5492, 5592, 5512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279609344 dispatch_meta_bytes=229704 combine_payload_bytes=279609344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279839048 ingress_bytes=389709824 link_bytes=389709824 token_bytes=559448392 weight_movement_bytes=110100480 total_nvlink_bytes=669548872 weight_to_token_ratio=0.196802 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777694802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278663168 dispatch_meta_bytes=229808 combine_payload_bytes=278663168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278892976 ingress_bytes=388763648 link_bytes=388763648 token_bytes=557556144 weight_movement_bytes=110100480 total_nvlink_bytes=667656624 weight_to_token_ratio=0.197470 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761239245e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278892544 dispatch_meta_bytes=228928 combine_payload_bytes=278892544 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279121472 ingress_bytes=388993024 link_bytes=388993024 token_bytes=558014016 weight_movement_bytes=110100480 total_nvlink_bytes=668114496 weight_to_token_ratio=0.197308 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765228471e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276297728 dispatch_meta_bytes=227424 combine_payload_bytes=276297728 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=276525152 ingress_bytes=496498688 link_bytes=496498688 token_bytes=552822880 weight_movement_bytes=220200960 total_nvlink_bytes=773023840 weight_to_token_ratio=0.398321 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.634928784e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277917696 dispatch_meta_bytes=229024 combine_payload_bytes=277917696 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278146720 ingress_bytes=498118656 link_bytes=498118656 token_bytes=556064416 weight_movement_bytes=220200960 total_nvlink_bytes=776265376 weight_to_token_ratio=0.395999 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.663102692e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38908 dup_rows=21002 dup_rows_remote=18324 dedup_ratio=0.320465 router_tokens_per_expert=[557, 521, 530, 511, 525, 495, 540, 467, 506, 525, 537, 501, 529, 535, 498, 521, 475, 531, 525, 508, 509, 560, 520, 494, 460, 536, 531, 529, 505, 494, 501, 532, 526, 527, 500, 526, 564, 509, 470, 479, 529, 505, 489, 473, 497, 493, 481, 543, 495, 521, 527, 513, 501, 489, 498, 533, 532, 534, 471, 528, 473, 492, 552, 490] payload_rows_per_dst=[5582, 5626, 5574, 5506, 5566, 5484, 5568, 5628] dup_rows_per_dst=[2710, 2678, 2670, 2670, 2636, 2536, 2586, 2516] combine_rows_per_src=[5582, 5626, 5574, 5506, 5566, 5484, 5568, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44470 payload_rows_remote=38998 dup_rows=21066 dup_rows_remote=18450 dedup_ratio=0.321442 router_tokens_per_expert=[561, 516, 497, 495, 536, 505, 504, 517, 506, 488, 540, 509, 522, 526, 523, 493, 503, 513, 509, 521, 518, 473, 555, 487, 500, 512, 509, 504, 530, 473, 509, 507, 493, 505, 517, 460, 544, 525, 503, 506, 537, 513, 531, 527, 495, 513, 497, 515, 542, 486, 491, 536, 517, 478, 525, 509, 486, 506, 525, 526, 473, 551, 525, 550] payload_rows_per_dst=[5580, 5592, 5540, 5472, 5560, 5618, 5568, 5540] dup_rows_per_dst=[2682, 2622, 2618, 2616, 2546, 2638, 2600, 2744] combine_rows_per_src=[5580, 5592, 5540, 5472, 5560, 5618, 5568, 5540] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278505472 dispatch_meta_bytes=229256 combine_payload_bytes=278505472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278734728 ingress_bytes=388605952 link_bytes=388605952 token_bytes=557240200 weight_movement_bytes=110100480 total_nvlink_bytes=667340680 weight_to_token_ratio=0.197582 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758496652e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276297728 dispatch_meta_bytes=227424 combine_payload_bytes=276297728 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=276525152 ingress_bytes=496498688 link_bytes=496498688 token_bytes=552822880 weight_movement_bytes=220200960 total_nvlink_bytes=773023840 weight_to_token_ratio=0.398321 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.634928784e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=229792 combine_payload_bytes=279537664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279767456 ingress_bytes=499738624 link_bytes=499738624 token_bytes=559305120 weight_movement_bytes=220200960 total_nvlink_bytes=779506080 weight_to_token_ratio=0.393705 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.691276600e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278548480 dispatch_meta_bytes=229880 combine_payload_bytes=278548480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278778360 ingress_bytes=388648960 link_bytes=388648960 token_bytes=557326840 weight_movement_bytes=110100480 total_nvlink_bytes=667427320 weight_to_token_ratio=0.197551 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759244632e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38902 dup_rows=21138 dup_rows_remote=18552 dedup_ratio=0.322540 router_tokens_per_expert=[490, 524, 510, 543, 532, 486, 512, 496, 515, 514, 515, 535, 539, 531, 492, 534, 487, 557, 502, 524, 519, 438, 524, 490, 531, 528, 536, 541, 479, 501, 480, 522, 543, 502, 590, 517, 482, 496, 504, 490, 493, 497, 493, 500, 529, 496, 528, 517, 484, 482, 498, 517, 544, 523, 502, 519, 499, 526, 514, 494, 501, 520, 531, 510] payload_rows_per_dst=[5492, 5646, 5496, 5574, 5546, 5494, 5584, 5566] dup_rows_per_dst=[2694, 2704, 2586, 2662, 2702, 2612, 2554, 2624] combine_rows_per_src=[5492, 5646, 5496, 5574, 5546, 5494, 5584, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44352 payload_rows_remote=38860 dup_rows=21184 dup_rows_remote=18610 dedup_ratio=0.323242 router_tokens_per_expert=[518, 526, 515, 523, 517, 497, 523, 513, 511, 529, 507, 530, 496, 504, 519, 504, 487, 509, 501, 501, 541, 515, 508, 515, 501, 514, 541, 538, 517, 516, 539, 512, 501, 521, 473, 514, 472, 516, 506, 538, 529, 508, 496, 494, 489, 539, 452, 526, 509, 502, 478, 511, 543, 513, 532, 495, 525, 501, 505, 498, 521, 553, 520, 501] payload_rows_per_dst=[5562, 5632, 5498, 5554, 5510, 5492, 5592, 5512] dup_rows_per_dst=[2702, 2568, 2656, 2802, 2572, 2574, 2574, 2736] combine_rows_per_src=[5562, 5632, 5498, 5554, 5510, 5492, 5592, 5512] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38854 dup_rows=21070 dup_rows_remote=18338 dedup_ratio=0.321503 router_tokens_per_expert=[494, 516, 508, 505, 497, 548, 582, 502, 512, 493, 489, 530, 497, 545, 493, 487, 525, 553, 515, 475, 535, 503, 556, 513, 507, 513, 500, 480, 533, 535, 505, 504, 483, 523, 469, 507, 461, 516, 498, 515, 517, 492, 526, 512, 505, 525, 494, 527, 517, 512, 516, 489, 544, 527, 479, 560, 507, 503, 505, 532, 543, 478, 510, 526] payload_rows_per_dst=[5612, 5550, 5608, 5542, 5428, 5850, 5604, 5272] dup_rows_per_dst=[2732, 2572, 2716, 2612, 2516, 3174, 2640, 2108] combine_rows_per_src=[5612, 5550, 5608, 5542, 5428, 5850, 5604, 5272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=39086 dup_rows=21000 dup_rows_remote=18424 dedup_ratio=0.320435 router_tokens_per_expert=[545, 531, 538, 531, 497, 494, 498, 520, 527, 506, 489, 463, 497, 502, 519, 510, 543, 528, 511, 491, 496, 484, 525, 522, 522, 504, 519, 492, 513, 515, 475, 502, 513, 515, 512, 533, 545, 521, 510, 519, 481, 513, 503, 497, 521, 554, 553, 567, 511, 501, 506, 515, 503, 508, 502, 474, 533, 487, 499, 495, 505, 511, 546, 506] payload_rows_per_dst=[5664, 5450, 5532, 5544, 5624, 5616, 5522, 5584] dup_rows_per_dst=[2644, 2576, 2668, 2540, 2712, 2762, 2518, 2580] combine_rows_per_src=[5664, 5450, 5532, 5544, 5624, 5616, 5522, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44578 payload_rows_remote=39046 dup_rows=20958 dup_rows_remote=18382 dedup_ratio=0.319794 router_tokens_per_expert=[532, 519, 509, 498, 489, 520, 510, 473, 482, 509, 512, 550, 501, 521, 491, 530, 502, 546, 509, 518, 517, 539, 527, 539, 531, 528, 541, 495, 556, 521, 487, 490, 514, 529, 514, 527, 505, 492, 494, 497, 528, 524, 501, 496, 495, 530, 522, 515, 491, 510, 515, 486, 523, 512, 509, 508, 512, 499, 482, 520, 518, 491, 505, 512] payload_rows_per_dst=[5516, 5574, 5676, 5582, 5594, 5562, 5532, 5542] dup_rows_per_dst=[2584, 2618, 2718, 2716, 2550, 2660, 2576, 2536] combine_rows_per_src=[5516, 5574, 5676, 5582, 5594, 5562, 5532, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44348 payload_rows_remote=38760 dup_rows=21188 dup_rows_remote=18648 dedup_ratio=0.323303 router_tokens_per_expert=[495, 520, 507, 460, 481, 511, 531, 534, 570, 482, 496, 536, 546, 503, 534, 530, 493, 466, 519, 514, 508, 546, 536, 507, 527, 499, 502, 532, 502, 499, 500, 486, 489, 487, 530, 489, 539, 483, 522, 525, 487, 532, 508, 530, 539, 508, 504, 528, 511, 535, 514, 519, 496, 508, 515, 501, 531, 521, 515, 500, 502, 510, 497, 521] payload_rows_per_dst=[5514, 5580, 5546, 5534, 5588, 5528, 5580, 5478] dup_rows_per_dst=[2564, 2814, 2632, 2560, 2540, 2744, 2618, 2716] combine_rows_per_src=[5514, 5580, 5546, 5534, 5588, 5528, 5580, 5478] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229816 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279079352 ingress_bytes=499050496 link_bytes=499050496 token_bytes=557928888 weight_movement_bytes=220200960 total_nvlink_bytes=778129848 weight_to_token_ratio=0.394676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679308922e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278892544 dispatch_meta_bytes=228928 combine_payload_bytes=278892544 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279121472 ingress_bytes=388993024 link_bytes=388993024 token_bytes=558014016 weight_movement_bytes=110100480 total_nvlink_bytes=668114496 weight_to_token_ratio=0.197308 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765228471e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=229792 combine_payload_bytes=279537664 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279767456 ingress_bytes=499738624 link_bytes=499738624 token_bytes=559305120 weight_movement_bytes=220200960 total_nvlink_bytes=779506080 weight_to_token_ratio=0.393705 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.691276600e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278505472 dispatch_meta_bytes=228768 combine_payload_bytes=278505472 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278734240 ingress_bytes=608806912 link_bytes=608806912 token_bytes=557239712 weight_movement_bytes=330301440 total_nvlink_bytes=887541152 weight_to_token_ratio=0.592746 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058815351e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278548480 dispatch_meta_bytes=229880 combine_payload_bytes=278548480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278778360 ingress_bytes=388648960 link_bytes=388648960 token_bytes=557326840 weight_movement_bytes=110100480 total_nvlink_bytes=667427320 weight_to_token_ratio=0.197551 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759244632e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44808 payload_rows_remote=39124 dup_rows=20728 dup_rows_remote=18042 dedup_ratio=0.316284 router_tokens_per_expert=[524, 512, 499, 501, 479, 526, 499, 511, 486, 479, 492, 513, 539, 517, 508, 519, 519, 519, 555, 504, 519, 552, 557, 527, 515, 514, 512, 512, 543, 510, 515, 503, 504, 520, 524, 549, 497, 542, 506, 498, 518, 535, 494, 516, 485, 482, 507, 527, 470, 562, 512, 511, 483, 514, 510, 514, 492, 503, 526, 470, 466, 485, 544, 522] payload_rows_per_dst=[5488, 5546, 5544, 5642, 5668, 5512, 5724, 5684] dup_rows_per_dst=[2442, 2560, 2416, 2606, 2612, 2616, 2790, 2686] combine_rows_per_src=[5488, 5546, 5544, 5642, 5668, 5512, 5724, 5684] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=38964 dup_rows=20990 dup_rows_remote=18262 dedup_ratio=0.320282 router_tokens_per_expert=[502, 509, 539, 484, 504, 523, 490, 543, 512, 477, 537, 510, 485, 501, 487, 523, 529, 510, 511, 538, 509, 527, 515, 532, 489, 520, 516, 500, 507, 551, 504, 490, 509, 505, 509, 493, 461, 530, 483, 513, 512, 522, 538, 501, 488, 509, 561, 524, 536, 528, 514, 517, 539, 472, 540, 485, 506, 543, 507, 499, 529, 503, 511, 507] payload_rows_per_dst=[5584, 5520, 5670, 5534, 5452, 5582, 5592, 5612] dup_rows_per_dst=[2604, 2544, 2672, 2620, 2554, 2728, 2670, 2598] combine_rows_per_src=[5584, 5520, 5670, 5534, 5452, 5582, 5592, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44578 payload_rows_remote=39046 dup_rows=20958 dup_rows_remote=18382 dedup_ratio=0.319794 router_tokens_per_expert=[532, 519, 509, 498, 489, 520, 510, 473, 482, 509, 512, 550, 501, 521, 491, 530, 502, 546, 509, 518, 517, 539, 527, 539, 531, 528, 541, 495, 556, 521, 487, 490, 514, 529, 514, 527, 505, 492, 494, 497, 528, 524, 501, 496, 495, 530, 522, 515, 491, 510, 515, 486, 523, 512, 509, 508, 512, 499, 482, 520, 518, 491, 505, 512] payload_rows_per_dst=[5516, 5574, 5676, 5582, 5594, 5562, 5532, 5542] dup_rows_per_dst=[2584, 2618, 2718, 2716, 2550, 2660, 2576, 2536] combine_rows_per_src=[5516, 5574, 5676, 5582, 5594, 5562, 5532, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44348 payload_rows_remote=38760 dup_rows=21188 dup_rows_remote=18648 dedup_ratio=0.323303 router_tokens_per_expert=[495, 520, 507, 460, 481, 511, 531, 534, 570, 482, 496, 536, 546, 503, 534, 530, 493, 466, 519, 514, 508, 546, 536, 507, 527, 499, 502, 532, 502, 499, 500, 486, 489, 487, 530, 489, 539, 483, 522, 525, 487, 532, 508, 530, 539, 508, 504, 528, 511, 535, 514, 519, 496, 508, 515, 501, 531, 521, 515, 500, 502, 510, 497, 521] payload_rows_per_dst=[5514, 5580, 5546, 5534, 5588, 5528, 5580, 5478] dup_rows_per_dst=[2564, 2814, 2632, 2560, 2540, 2744, 2618, 2716] combine_rows_per_src=[5514, 5580, 5546, 5534, 5588, 5528, 5580, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38924 dup_rows=21026 dup_rows_remote=18420 dedup_ratio=0.320831 router_tokens_per_expert=[518, 512, 470, 537, 526, 527, 504, 487, 490, 515, 480, 496, 519, 491, 517, 516, 503, 533, 523, 515, 507, 515, 512, 488, 505, 532, 494, 506, 529, 502, 469, 522, 479, 540, 535, 509, 505, 502, 577, 577, 520, 529, 504, 545, 535, 474, 531, 498, 539, 474, 515, 470, 497, 513, 525, 524, 539, 532, 535, 488, 524, 479, 523, 471] payload_rows_per_dst=[5578, 5494, 5586, 5564, 5632, 5608, 5494, 5554] dup_rows_per_dst=[2584, 2554, 2606, 2554, 2816, 2664, 2620, 2628] combine_rows_per_src=[5578, 5494, 5586, 5564, 5632, 5608, 5494, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38884 dup_rows=21088 dup_rows_remote=18384 dedup_ratio=0.321777 router_tokens_per_expert=[524, 497, 519, 525, 505, 516, 543, 514, 539, 495, 503, 540, 517, 484, 542, 533, 508, 554, 518, 511, 537, 529, 527, 516, 542, 518, 530, 499, 547, 516, 473, 509, 517, 497, 511, 515, 491, 482, 523, 486, 534, 517, 512, 527, 515, 479, 463, 500, 482, 492, 548, 477, 506, 494, 483, 506, 530, 493, 522, 484, 534, 502, 523, 493] payload_rows_per_dst=[5626, 5590, 5632, 5564, 5532, 5450, 5508, 5546] dup_rows_per_dst=[2660, 2716, 2768, 2704, 2512, 2644, 2468, 2616] combine_rows_per_src=[5626, 5590, 5632, 5564, 5532, 5450, 5508, 5546] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280168448 dispatch_meta_bytes=230040 combine_payload_bytes=280168448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280398488 ingress_bytes=390268928 link_bytes=390268928 token_bytes=560566936 weight_movement_bytes=110100480 total_nvlink_bytes=670667416 weight_to_token_ratio=0.196409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787418540e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=39086 dup_rows=21000 dup_rows_remote=18424 dedup_ratio=0.320435 router_tokens_per_expert=[545, 531, 538, 531, 497, 494, 498, 520, 527, 506, 489, 463, 497, 502, 519, 510, 543, 528, 511, 491, 496, 484, 525, 522, 522, 504, 519, 492, 513, 515, 475, 502, 513, 515, 512, 533, 545, 521, 510, 519, 481, 513, 503, 497, 521, 554, 553, 567, 511, 501, 506, 515, 503, 508, 502, 474, 533, 487, 499, 495, 505, 511, 546, 506] payload_rows_per_dst=[5664, 5450, 5532, 5544, 5624, 5616, 5522, 5584] dup_rows_per_dst=[2644, 2576, 2668, 2540, 2712, 2762, 2518, 2580] combine_rows_per_src=[5664, 5450, 5532, 5544, 5624, 5616, 5522, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280440832 dispatch_meta_bytes=228664 combine_payload_bytes=280440832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280669496 ingress_bytes=390541312 link_bytes=390541312 token_bytes=561110328 weight_movement_bytes=110100480 total_nvlink_bytes=671210808 weight_to_token_ratio=0.196219 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.792155746e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=229712 combine_payload_bytes=279881728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280111440 ingress_bytes=389982208 link_bytes=389982208 token_bytes=559993168 weight_movement_bytes=110100480 total_nvlink_bytes=670093648 weight_to_token_ratio=0.196610 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782432008e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38884 dup_rows=21088 dup_rows_remote=18384 dedup_ratio=0.321777 router_tokens_per_expert=[524, 497, 519, 525, 505, 516, 543, 514, 539, 495, 503, 540, 517, 484, 542, 533, 508, 554, 518, 511, 537, 529, 527, 516, 542, 518, 530, 499, 547, 516, 473, 509, 517, 497, 511, 515, 491, 482, 523, 486, 534, 517, 512, 527, 515, 479, 463, 500, 482, 492, 548, 477, 506, 494, 483, 506, 530, 493, 522, 484, 534, 502, 523, 493] payload_rows_per_dst=[5626, 5590, 5632, 5564, 5532, 5450, 5508, 5546] dup_rows_per_dst=[2660, 2716, 2768, 2704, 2512, 2644, 2468, 2616] combine_rows_per_src=[5626, 5590, 5632, 5564, 5532, 5450, 5508, 5546] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=229632 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278061312 ingress_bytes=387932160 link_bytes=387932160 token_bytes=555892992 weight_movement_bytes=110100480 total_nvlink_bytes=665993472 weight_to_token_ratio=0.198061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746778301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280440832 dispatch_meta_bytes=228664 combine_payload_bytes=280440832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280669496 ingress_bytes=390541312 link_bytes=390541312 token_bytes=561110328 weight_movement_bytes=110100480 total_nvlink_bytes=671210808 weight_to_token_ratio=0.196219 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.792155746e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=228904 combine_payload_bytes=279293952 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279522856 ingress_bytes=609595392 link_bytes=609595392 token_bytes=558816808 weight_movement_bytes=330301440 total_nvlink_bytes=889118248 weight_to_token_ratio=0.591073 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060186648e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229072 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278949584 ingress_bytes=388820992 link_bytes=388820992 token_bytes=557670096 weight_movement_bytes=110100480 total_nvlink_bytes=667770576 weight_to_token_ratio=0.197429 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762236552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38810 dup_rows=21082 dup_rows_remote=18392 dedup_ratio=0.321686 router_tokens_per_expert=[511, 508, 482, 543, 523, 524, 515, 534, 502, 503, 526, 515, 496, 496, 505, 506, 516, 533, 491, 507, 526, 523, 525, 520, 460, 519, 511, 488, 532, 502, 505, 522, 564, 524, 502, 535, 504, 504, 464, 525, 517, 504, 533, 502, 499, 562, 529, 453, 541, 496, 519, 485, 526, 509, 487, 512, 507, 488, 499, 543, 515, 517, 506, 528] payload_rows_per_dst=[5644, 5566, 5498, 5544, 5664, 5396, 5554, 5588] dup_rows_per_dst=[2690, 2828, 2488, 2584, 2910, 2368, 2596, 2618] combine_rows_per_src=[5644, 5566, 5498, 5544, 5664, 5396, 5554, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38906 dup_rows=21118 dup_rows_remote=18554 dedup_ratio=0.322235 router_tokens_per_expert=[523, 470, 511, 538, 556, 507, 541, 510, 463, 504, 499, 534, 523, 523, 494, 487, 526, 486, 501, 533, 486, 504, 474, 528, 536, 516, 498, 551, 521, 508, 515, 486, 505, 502, 505, 495, 494, 520, 462, 529, 505, 535, 535, 516, 533, 512, 533, 515, 517, 494, 553, 503, 506, 527, 524, 520, 493, 501, 506, 472, 528, 514, 535, 527] payload_rows_per_dst=[5582, 5546, 5512, 5624, 5462, 5558, 5552, 5582] dup_rows_per_dst=[2730, 2508, 2564, 2638, 2562, 2810, 2736, 2570] combine_rows_per_src=[5582, 5546, 5512, 5624, 5462, 5558, 5552, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=38964 dup_rows=20990 dup_rows_remote=18262 dedup_ratio=0.320282 router_tokens_per_expert=[502, 509, 539, 484, 504, 523, 490, 543, 512, 477, 537, 510, 485, 501, 487, 523, 529, 510, 511, 538, 509, 527, 515, 532, 489, 520, 516, 500, 507, 551, 504, 490, 509, 505, 509, 493, 461, 530, 483, 513, 512, 522, 538, 501, 488, 509, 561, 524, 536, 528, 514, 517, 539, 472, 540, 485, 506, 543, 507, 499, 529, 503, 511, 507] payload_rows_per_dst=[5584, 5520, 5670, 5534, 5452, 5582, 5592, 5612] dup_rows_per_dst=[2604, 2544, 2672, 2620, 2554, 2728, 2670, 2598] combine_rows_per_src=[5584, 5520, 5670, 5534, 5452, 5582, 5592, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229376 combine_payload_bytes=279007232 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279236608 ingress_bytes=609308672 link_bytes=609308672 token_bytes=558243840 weight_movement_bytes=330301440 total_nvlink_bytes=888545280 weight_to_token_ratio=0.591680 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059687995e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280168448 dispatch_meta_bytes=230040 combine_payload_bytes=280168448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280398488 ingress_bytes=390268928 link_bytes=390268928 token_bytes=560566936 weight_movement_bytes=110100480 total_nvlink_bytes=670667416 weight_to_token_ratio=0.196409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787418540e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39046 dup_rows=20988 dup_rows_remote=18246 dedup_ratio=0.320251 router_tokens_per_expert=[499, 499, 553, 505, 560, 479, 495, 521, 523, 549, 505, 523, 520, 534, 486, 482, 479, 498, 509, 505, 525, 519, 545, 510, 503, 517, 501, 511, 506, 533, 537, 502, 476, 496, 489, 546, 471, 465, 504, 514, 473, 472, 525, 548, 470, 552, 483, 511, 506, 487, 521, 498, 539, 556, 524, 515, 504, 493, 538, 522, 525, 522, 537, 553] payload_rows_per_dst=[5630, 5502, 5514, 5540, 5480, 5574, 5632, 5676] dup_rows_per_dst=[2592, 2742, 2666, 2680, 2442, 2494, 2660, 2712] combine_rows_per_src=[5630, 5502, 5514, 5540, 5480, 5574, 5632, 5676] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44844 payload_rows_remote=39176 dup_rows=20692 dup_rows_remote=18014 dedup_ratio=0.315735 router_tokens_per_expert=[516, 503, 502, 529, 486, 499, 520, 491, 520, 499, 514, 526, 535, 518, 523, 505, 491, 498, 528, 528, 516, 519, 520, 468, 501, 509, 540, 487, 480, 546, 511, 524, 523, 536, 515, 459, 517, 490, 496, 513, 517, 428, 496, 506, 509, 545, 496, 529, 517, 500, 539, 502, 513, 529, 523, 550, 556, 516, 505, 504, 541, 504, 520, 522] payload_rows_per_dst=[5550, 5656, 5578, 5610, 5498, 5578, 5668, 5706] dup_rows_per_dst=[2542, 2624, 2558, 2586, 2600, 2474, 2678, 2630] combine_rows_per_src=[5550, 5656, 5578, 5610, 5498, 5578, 5668, 5706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229072 combine_payload_bytes=278720512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278949584 ingress_bytes=388820992 link_bytes=388820992 token_bytes=557670096 weight_movement_bytes=110100480 total_nvlink_bytes=667770576 weight_to_token_ratio=0.197429 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762236552e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278190080 dispatch_meta_bytes=228808 combine_payload_bytes=278190080 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278418888 ingress_bytes=498391040 link_bytes=498391040 token_bytes=556608968 weight_movement_bytes=220200960 total_nvlink_bytes=776809928 weight_to_token_ratio=0.395612 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.667839898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38892 dup_rows=21072 dup_rows_remote=18388 dedup_ratio=0.321533 router_tokens_per_expert=[541, 513, 487, 493, 507, 518, 534, 527, 516, 501, 504, 483, 506, 515, 508, 516, 482, 500, 512, 549, 543, 542, 506, 529, 475, 487, 556, 531, 521, 496, 509, 516, 540, 524, 499, 461, 476, 546, 543, 539, 456, 549, 512, 505, 542, 485, 532, 489, 508, 516, 510, 502, 556, 514, 483, 503, 524, 511, 498, 519, 508, 471, 515, 509] payload_rows_per_dst=[5572, 5502, 5632, 5602, 5572, 5502, 5588, 5494] dup_rows_per_dst=[2668, 2596, 2694, 2580, 2684, 2638, 2596, 2616] combine_rows_per_src=[5572, 5502, 5632, 5602, 5572, 5502, 5588, 5494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=229840 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279108048 ingress_bytes=719280128 link_bytes=719280128 token_bytes=557986256 weight_movement_bytes=440401920 total_nvlink_bytes=998388176 weight_to_token_ratio=0.789270 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250946444e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=228904 combine_payload_bytes=279293952 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279522856 ingress_bytes=609595392 link_bytes=609595392 token_bytes=558816808 weight_movement_bytes=330301440 total_nvlink_bytes=889118248 weight_to_token_ratio=0.591073 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060186648e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38894 dup_rows=21090 dup_rows_remote=18386 dedup_ratio=0.321808 router_tokens_per_expert=[518, 516, 525, 525, 475, 536, 519, 481, 516, 491, 552, 507, 478, 525, 527, 532, 500, 519, 520, 504, 505, 540, 508, 547, 516, 511, 526, 511, 507, 504, 490, 520, 471, 483, 544, 515, 502, 472, 531, 491, 503, 534, 509, 526, 489, 499, 545, 511, 546, 526, 515, 498, 485, 492, 509, 512, 530, 502, 480, 515, 488, 522, 536, 536] payload_rows_per_dst=[5516, 5530, 5462, 5536, 5640, 5638, 5572, 5552] dup_rows_per_dst=[2674, 2726, 2338, 2634, 2864, 2594, 2556, 2704] combine_rows_per_src=[5516, 5530, 5462, 5536, 5640, 5638, 5572, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44844 payload_rows_remote=39176 dup_rows=20692 dup_rows_remote=18014 dedup_ratio=0.315735 router_tokens_per_expert=[516, 503, 502, 529, 486, 499, 520, 491, 520, 499, 514, 526, 535, 518, 523, 505, 491, 498, 528, 528, 516, 519, 520, 468, 501, 509, 540, 487, 480, 546, 511, 524, 523, 536, 515, 459, 517, 490, 496, 513, 517, 428, 496, 506, 509, 545, 496, 529, 517, 500, 539, 502, 513, 529, 523, 550, 556, 516, 505, 504, 541, 504, 520, 522] payload_rows_per_dst=[5550, 5656, 5578, 5610, 5498, 5578, 5668, 5706] dup_rows_per_dst=[2542, 2624, 2558, 2586, 2600, 2474, 2678, 2630] combine_rows_per_src=[5550, 5656, 5578, 5610, 5498, 5578, 5668, 5706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38892 dup_rows=21072 dup_rows_remote=18388 dedup_ratio=0.321533 router_tokens_per_expert=[541, 513, 487, 493, 507, 518, 534, 527, 516, 501, 504, 483, 506, 515, 508, 516, 482, 500, 512, 549, 543, 542, 506, 529, 475, 487, 556, 531, 521, 496, 509, 516, 540, 524, 499, 461, 476, 546, 543, 539, 456, 549, 512, 505, 542, 485, 532, 489, 508, 516, 510, 502, 556, 514, 483, 503, 524, 511, 498, 519, 508, 471, 515, 509] payload_rows_per_dst=[5572, 5502, 5632, 5602, 5572, 5502, 5588, 5494] dup_rows_per_dst=[2668, 2596, 2694, 2580, 2684, 2638, 2596, 2616] combine_rows_per_src=[5572, 5502, 5632, 5602, 5572, 5502, 5588, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38966 dup_rows=21142 dup_rows_remote=18474 dedup_ratio=0.322601 router_tokens_per_expert=[493, 520, 459, 491, 514, 487, 531, 532, 523, 546, 547, 460, 489, 514, 534, 561, 551, 533, 513, 530, 486, 504, 510, 516, 521, 533, 536, 536, 489, 491, 504, 516, 508, 518, 498, 518, 497, 489, 528, 524, 498, 487, 494, 517, 499, 523, 522, 508, 498, 489, 460, 516, 492, 514, 514, 488, 566, 519, 508, 506, 524, 555, 506, 515] payload_rows_per_dst=[5506, 5662, 5592, 5574, 5588, 5428, 5424, 5620] dup_rows_per_dst=[2548, 2686, 2694, 2678, 2572, 2668, 2518, 2778] combine_rows_per_src=[5506, 5662, 5592, 5574, 5588, 5428, 5424, 5620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44354 payload_rows_remote=38920 dup_rows=21182 dup_rows_remote=18596 dedup_ratio=0.323212 router_tokens_per_expert=[512, 504, 528, 556, 502, 496, 512, 519, 530, 534, 490, 509, 502, 477, 484, 527, 523, 511, 515, 521, 528, 532, 516, 521, 514, 453, 548, 492, 453, 499, 503, 548, 558, 516, 515, 498, 526, 490, 515, 503, 500, 543, 538, 532, 542, 517, 524, 520, 545, 496, 531, 474, 513, 508, 492, 530, 511, 516, 486, 485, 511, 511, 489, 474] payload_rows_per_dst=[5594, 5506, 5652, 5434, 5550, 5598, 5510, 5510] dup_rows_per_dst=[2664, 2600, 2682, 2586, 2692, 2834, 2668, 2456] combine_rows_per_src=[5594, 5506, 5652, 5434, 5550, 5598, 5510, 5510] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=229168 combine_payload_bytes=279881728 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280110896 ingress_bytes=500082688 link_bytes=500082688 token_bytes=559992624 weight_movement_bytes=220200960 total_nvlink_bytes=780193584 weight_to_token_ratio=0.393221 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697260439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39046 dup_rows=20988 dup_rows_remote=18246 dedup_ratio=0.320251 router_tokens_per_expert=[499, 499, 553, 505, 560, 479, 495, 521, 523, 549, 505, 523, 520, 534, 486, 482, 479, 498, 509, 505, 525, 519, 545, 510, 503, 517, 501, 511, 506, 533, 537, 502, 476, 496, 489, 546, 471, 465, 504, 514, 473, 472, 525, 548, 470, 552, 483, 511, 506, 487, 521, 498, 539, 556, 524, 515, 504, 493, 538, 522, 525, 522, 537, 553] payload_rows_per_dst=[5630, 5502, 5514, 5540, 5480, 5574, 5632, 5676] dup_rows_per_dst=[2592, 2742, 2666, 2680, 2442, 2494, 2660, 2712] combine_rows_per_src=[5630, 5502, 5514, 5540, 5480, 5574, 5632, 5676] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38902 dup_rows=21138 dup_rows_remote=18552 dedup_ratio=0.322540 router_tokens_per_expert=[490, 524, 510, 543, 532, 486, 512, 496, 515, 514, 515, 535, 539, 531, 492, 534, 487, 557, 502, 524, 519, 438, 524, 490, 531, 528, 536, 541, 479, 501, 480, 522, 543, 502, 590, 517, 482, 496, 504, 490, 493, 497, 493, 500, 529, 496, 528, 517, 484, 482, 498, 517, 544, 523, 502, 519, 499, 526, 514, 494, 501, 520, 531, 510] payload_rows_per_dst=[5492, 5646, 5496, 5574, 5546, 5494, 5584, 5566] dup_rows_per_dst=[2694, 2704, 2586, 2662, 2702, 2612, 2554, 2624] combine_rows_per_src=[5492, 5646, 5496, 5574, 5546, 5494, 5584, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=229120 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279021312 ingress_bytes=498993152 link_bytes=498993152 token_bytes=557813504 weight_movement_bytes=220200960 total_nvlink_bytes=778014464 weight_to_token_ratio=0.394757 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678311616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44354 payload_rows_remote=38920 dup_rows=21182 dup_rows_remote=18596 dedup_ratio=0.323212 router_tokens_per_expert=[512, 504, 528, 556, 502, 496, 512, 519, 530, 534, 490, 509, 502, 477, 484, 527, 523, 511, 515, 521, 528, 532, 516, 521, 514, 453, 548, 492, 453, 499, 503, 548, 558, 516, 515, 498, 526, 490, 515, 503, 500, 543, 538, 532, 542, 517, 524, 520, 545, 496, 531, 474, 513, 508, 492, 530, 511, 516, 486, 485, 511, 511, 489, 474] payload_rows_per_dst=[5594, 5506, 5652, 5434, 5550, 5598, 5510, 5510] dup_rows_per_dst=[2664, 2600, 2682, 2586, 2692, 2834, 2668, 2456] combine_rows_per_src=[5594, 5506, 5652, 5434, 5550, 5598, 5510, 5510] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280813568 dispatch_meta_bytes=228760 combine_payload_bytes=280813568 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=281042328 ingress_bytes=721215488 link_bytes=721215488 token_bytes=561855896 weight_movement_bytes=440401920 total_nvlink_bytes=1002257816 weight_to_token_ratio=0.783834 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.254312353e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=229120 combine_payload_bytes=278777856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279006976 ingress_bytes=388878336 link_bytes=388878336 token_bytes=557784832 weight_movement_bytes=110100480 total_nvlink_bytes=667885312 weight_to_token_ratio=0.197389 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763233858e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=229120 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279021312 ingress_bytes=498993152 link_bytes=498993152 token_bytes=557813504 weight_movement_bytes=220200960 total_nvlink_bytes=778014464 weight_to_token_ratio=0.394757 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678311616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=229760 combine_payload_bytes=279308288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279538048 ingress_bytes=389408768 link_bytes=389408768 token_bytes=558846336 weight_movement_bytes=110100480 total_nvlink_bytes=668946816 weight_to_token_ratio=0.197014 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772458943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278978560 dispatch_meta_bytes=230064 combine_payload_bytes=278978560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279208624 ingress_bytes=389079040 link_bytes=389079040 token_bytes=558187184 weight_movement_bytes=110100480 total_nvlink_bytes=668287664 weight_to_token_ratio=0.197247 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766724431e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44342 payload_rows_remote=38802 dup_rows=21194 dup_rows_remote=18460 dedup_ratio=0.323395 router_tokens_per_expert=[467, 467, 535, 476, 536, 489, 511, 498, 513, 518, 537, 513, 506, 527, 524, 529, 527, 539, 539, 505, 503, 498, 480, 521, 480, 500, 498, 477, 505, 517, 509, 492, 489, 540, 495, 507, 513, 500, 482, 515, 492, 525, 512, 534, 527, 527, 515, 503, 520, 557, 508, 518, 525, 534, 491, 571, 530, 508, 508, 525, 517, 525, 493, 526] payload_rows_per_dst=[5540, 5452, 5590, 5590, 5526, 5560, 5590, 5494] dup_rows_per_dst=[2734, 2606, 2664, 2710, 2556, 2794, 2704, 2426] combine_rows_per_src=[5540, 5452, 5590, 5590, 5526, 5560, 5590, 5494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44620 payload_rows_remote=39066 dup_rows=20916 dup_rows_remote=18250 dedup_ratio=0.319153 router_tokens_per_expert=[520, 505, 536, 516, 486, 535, 522, 483, 536, 519, 502, 496, 486, 511, 526, 497, 546, 453, 526, 528, 542, 497, 526, 492, 526, 516, 471, 492, 526, 551, 497, 519, 482, 510, 482, 527, 518, 553, 521, 543, 480, 529, 492, 511, 532, 522, 502, 513, 532, 500, 495, 487, 502, 502, 552, 516, 531, 500, 539, 518, 484, 486, 512, 511] payload_rows_per_dst=[5622, 5586, 5554, 5596, 5624, 5500, 5564, 5574] dup_rows_per_dst=[2584, 2560, 2666, 2600, 2648, 2662, 2608, 2588] combine_rows_per_src=[5622, 5586, 5554, 5596, 5624, 5500, 5564, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38966 dup_rows=21142 dup_rows_remote=18474 dedup_ratio=0.322601 router_tokens_per_expert=[493, 520, 459, 491, 514, 487, 531, 532, 523, 546, 547, 460, 489, 514, 534, 561, 551, 533, 513, 530, 486, 504, 510, 516, 521, 533, 536, 536, 489, 491, 504, 516, 508, 518, 498, 518, 497, 489, 528, 524, 498, 487, 494, 517, 499, 523, 522, 508, 498, 489, 460, 516, 492, 514, 514, 488, 566, 519, 508, 506, 524, 555, 506, 515] payload_rows_per_dst=[5506, 5662, 5592, 5574, 5588, 5428, 5424, 5620] dup_rows_per_dst=[2548, 2686, 2694, 2678, 2572, 2668, 2518, 2778] combine_rows_per_src=[5506, 5662, 5592, 5574, 5588, 5428, 5424, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=229168 combine_payload_bytes=279881728 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280110896 ingress_bytes=500082688 link_bytes=500082688 token_bytes=559992624 weight_movement_bytes=220200960 total_nvlink_bytes=780193584 weight_to_token_ratio=0.393221 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697260439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44540 payload_rows_remote=38938 dup_rows=20996 dup_rows_remote=18360 dedup_ratio=0.320374 router_tokens_per_expert=[515, 517, 548, 504, 495, 502, 499, 540, 521, 522, 535, 495, 483, 520, 529, 514, 516, 490, 529, 483, 491, 502, 489, 505, 497, 519, 522, 528, 491, 518, 488, 504, 499, 517, 526, 512, 495, 531, 534, 534, 539, 479, 510, 498, 500, 522, 456, 499, 524, 512, 524, 498, 503, 507, 513, 513, 529, 496, 492, 540, 540, 561, 497, 557] payload_rows_per_dst=[5538, 5602, 5538, 5534, 5588, 5518, 5544, 5678] dup_rows_per_dst=[2702, 2636, 2472, 2600, 2708, 2488, 2644, 2746] combine_rows_per_src=[5538, 5602, 5538, 5534, 5588, 5518, 5544, 5678] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44338 payload_rows_remote=38922 dup_rows=21198 dup_rows_remote=18656 dedup_ratio=0.323456 router_tokens_per_expert=[511, 479, 518, 506, 538, 480, 506, 513, 536, 514, 509, 546, 537, 516, 515, 506, 502, 550, 501, 567, 511, 511, 517, 476, 536, 491, 496, 545, 509, 539, 472, 541, 516, 542, 498, 509, 541, 464, 513, 553, 509, 490, 495, 528, 500, 496, 514, 511, 507, 497, 496, 502, 467, 495, 487, 528, 531, 537, 542, 511, 472, 480, 508, 535] payload_rows_per_dst=[5506, 5630, 5590, 5540, 5530, 5530, 5416, 5596] dup_rows_per_dst=[2596, 2728, 2680, 2718, 2742, 2556, 2542, 2636] combine_rows_per_src=[5506, 5630, 5590, 5540, 5530, 5530, 5416, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229816 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279079352 ingress_bytes=499050496 link_bytes=499050496 token_bytes=557928888 weight_movement_bytes=220200960 total_nvlink_bytes=778129848 weight_to_token_ratio=0.394676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679308922e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278978560 dispatch_meta_bytes=230064 combine_payload_bytes=278978560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279208624 ingress_bytes=389079040 link_bytes=389079040 token_bytes=558187184 weight_movement_bytes=110100480 total_nvlink_bytes=668287664 weight_to_token_ratio=0.197247 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766724431e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278132736 dispatch_meta_bytes=229048 combine_payload_bytes=278132736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278361784 ingress_bytes=388233216 link_bytes=388233216 token_bytes=556494520 weight_movement_bytes=110100480 total_nvlink_bytes=666595000 weight_to_token_ratio=0.197846 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752014160e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=229264 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280254352 ingress_bytes=500226048 link_bytes=500226048 token_bytes=560279440 weight_movement_bytes=220200960 total_nvlink_bytes=780480400 weight_to_token_ratio=0.393020 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.699753705e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44698 payload_rows_remote=39104 dup_rows=20838 dup_rows_remote=18152 dedup_ratio=0.317963 router_tokens_per_expert=[488, 507, 498, 488, 501, 546, 536, 525, 490, 508, 490, 522, 545, 472, 538, 502, 551, 531, 495, 558, 522, 446, 484, 559, 500, 494, 519, 521, 478, 488, 472, 472, 486, 536, 496, 523, 520, 533, 502, 544, 538, 502, 534, 496, 545, 504, 513, 518, 512, 539, 517, 551, 512, 524, 519, 503, 518, 503, 490, 469, 544, 481, 540, 510] payload_rows_per_dst=[5574, 5550, 5604, 5426, 5594, 5622, 5680, 5648] dup_rows_per_dst=[2604, 2584, 2688, 2462, 2686, 2678, 2674, 2462] combine_rows_per_src=[5574, 5550, 5604, 5426, 5594, 5622, 5680, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279308288 dispatch_meta_bytes=229760 combine_payload_bytes=279308288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279538048 ingress_bytes=389408768 link_bytes=389408768 token_bytes=558846336 weight_movement_bytes=110100480 total_nvlink_bytes=668946816 weight_to_token_ratio=0.197014 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772458943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38902 dup_rows=21034 dup_rows_remote=18316 dedup_ratio=0.320953 router_tokens_per_expert=[515, 511, 498, 482, 506, 470, 493, 543, 528, 543, 511, 511, 529, 505, 511, 533, 498, 515, 532, 473, 517, 519, 515, 523, 498, 511, 518, 466, 512, 516, 510, 549, 479, 542, 492, 495, 555, 491, 504, 528, 524, 552, 563, 532, 478, 501, 523, 498, 483, 501, 530, 502, 522, 474, 470, 509, 502, 525, 494, 533, 510, 532, 533, 530] payload_rows_per_dst=[5486, 5410, 5598, 5814, 5484, 5670, 5440, 5600] dup_rows_per_dst=[2514, 2468, 2622, 2810, 2466, 2894, 2542, 2718] combine_rows_per_src=[5486, 5410, 5598, 5814, 5484, 5670, 5440, 5600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44424 payload_rows_remote=38788 dup_rows=21112 dup_rows_remote=18494 dedup_ratio=0.322144 router_tokens_per_expert=[526, 499, 496, 485, 538, 494, 544, 490, 506, 510, 544, 519, 473, 461, 509, 541, 462, 493, 541, 488, 520, 569, 521, 492, 509, 473, 537, 561, 504, 475, 506, 543, 550, 514, 525, 526, 476, 524, 549, 496, 515, 522, 522, 511, 517, 511, 503, 526, 525, 486, 524, 504, 511, 503, 540, 497, 536, 488, 481, 492, 479, 545, 513, 528] payload_rows_per_dst=[5564, 5528, 5506, 5550, 5592, 5636, 5490, 5558] dup_rows_per_dst=[2580, 2598, 2666, 2666, 2728, 2618, 2690, 2566] combine_rows_per_src=[5564, 5528, 5506, 5550, 5592, 5636, 5490, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229192 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279336776 ingress_bytes=499308544 link_bytes=499308544 token_bytes=558444360 weight_movement_bytes=220200960 total_nvlink_bytes=778645320 weight_to_token_ratio=0.394311 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683796802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44338 payload_rows_remote=38922 dup_rows=21198 dup_rows_remote=18656 dedup_ratio=0.323456 router_tokens_per_expert=[511, 479, 518, 506, 538, 480, 506, 513, 536, 514, 509, 546, 537, 516, 515, 506, 502, 550, 501, 567, 511, 511, 517, 476, 536, 491, 496, 545, 509, 539, 472, 541, 516, 542, 498, 509, 541, 464, 513, 553, 509, 490, 495, 528, 500, 496, 514, 511, 507, 497, 496, 502, 467, 495, 487, 528, 531, 537, 542, 511, 472, 480, 508, 535] payload_rows_per_dst=[5506, 5630, 5590, 5540, 5530, 5530, 5416, 5596] dup_rows_per_dst=[2596, 2728, 2680, 2718, 2742, 2556, 2542, 2636] combine_rows_per_src=[5506, 5630, 5590, 5540, 5530, 5530, 5416, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44698 payload_rows_remote=39104 dup_rows=20838 dup_rows_remote=18152 dedup_ratio=0.317963 router_tokens_per_expert=[488, 507, 498, 488, 501, 546, 536, 525, 490, 508, 490, 522, 545, 472, 538, 502, 551, 531, 495, 558, 522, 446, 484, 559, 500, 494, 519, 521, 478, 488, 472, 472, 486, 536, 496, 523, 520, 533, 502, 544, 538, 502, 534, 496, 545, 504, 513, 518, 512, 539, 517, 551, 512, 524, 519, 503, 518, 503, 490, 469, 544, 481, 540, 510] payload_rows_per_dst=[5574, 5550, 5604, 5426, 5594, 5622, 5680, 5648] dup_rows_per_dst=[2604, 2584, 2688, 2462, 2686, 2678, 2674, 2462] combine_rows_per_src=[5574, 5550, 5604, 5426, 5594, 5622, 5680, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44540 payload_rows_remote=38938 dup_rows=20996 dup_rows_remote=18360 dedup_ratio=0.320374 router_tokens_per_expert=[515, 517, 548, 504, 495, 502, 499, 540, 521, 522, 535, 495, 483, 520, 529, 514, 516, 490, 529, 483, 491, 502, 489, 505, 497, 519, 522, 528, 491, 518, 488, 504, 499, 517, 526, 512, 495, 531, 534, 534, 539, 479, 510, 498, 500, 522, 456, 499, 524, 512, 524, 498, 503, 507, 513, 513, 529, 496, 492, 540, 540, 561, 497, 557] payload_rows_per_dst=[5538, 5602, 5538, 5534, 5588, 5518, 5544, 5678] dup_rows_per_dst=[2702, 2636, 2472, 2600, 2708, 2488, 2644, 2746] combine_rows_per_src=[5538, 5602, 5538, 5534, 5588, 5518, 5544, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44676 payload_rows_remote=39070 dup_rows=20860 dup_rows_remote=18154 dedup_ratio=0.318298 router_tokens_per_expert=[547, 542, 478, 506, 537, 537, 505, 540, 506, 556, 527, 512, 522, 500, 504, 491, 541, 500, 528, 498, 494, 496, 494, 535, 538, 546, 509, 507, 555, 489, 495, 517, 466, 489, 493, 534, 515, 526, 525, 486, 535, 513, 521, 467, 528, 517, 505, 480, 531, 543, 509, 514, 514, 516, 500, 509, 493, 530, 467, 493, 567, 493, 447, 490] payload_rows_per_dst=[5700, 5602, 5642, 5606, 5512, 5532, 5592, 5490] dup_rows_per_dst=[2684, 2634, 2530, 2706, 2556, 2600, 2680, 2470] combine_rows_per_src=[5700, 5602, 5642, 5606, 5512, 5532, 5592, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38892 dup_rows=21108 dup_rows_remote=18516 dedup_ratio=0.322083 router_tokens_per_expert=[521, 526, 500, 494, 485, 493, 536, 509, 476, 533, 542, 490, 506, 552, 505, 550, 485, 508, 491, 514, 556, 505, 493, 524, 487, 520, 504, 510, 496, 532, 457, 496, 475, 483, 515, 496, 555, 525, 511, 532, 494, 510, 554, 495, 537, 470, 463, 501, 537, 527, 542, 506, 513, 555, 504, 525, 525, 525, 487, 534, 514, 527, 510, 525] payload_rows_per_dst=[5536, 5654, 5536, 5494, 5560, 5444, 5632, 5572] dup_rows_per_dst=[2592, 2712, 2616, 2510, 2644, 2604, 2786, 2644] combine_rows_per_src=[5536, 5654, 5536, 5494, 5560, 5444, 5632, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38906 dup_rows=21118 dup_rows_remote=18554 dedup_ratio=0.322235 router_tokens_per_expert=[523, 470, 511, 538, 556, 507, 541, 510, 463, 504, 499, 534, 523, 523, 494, 487, 526, 486, 501, 533, 486, 504, 474, 528, 536, 516, 498, 551, 521, 508, 515, 486, 505, 502, 505, 495, 494, 520, 462, 529, 505, 535, 535, 516, 533, 512, 533, 515, 517, 494, 553, 503, 506, 527, 524, 520, 493, 501, 506, 472, 528, 514, 535, 527] payload_rows_per_dst=[5582, 5546, 5512, 5624, 5462, 5558, 5552, 5582] dup_rows_per_dst=[2730, 2508, 2564, 2638, 2562, 2810, 2736, 2570] combine_rows_per_src=[5582, 5546, 5512, 5624, 5462, 5558, 5552, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=228872 combine_payload_bytes=278849536 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279078408 ingress_bytes=499050496 link_bytes=499050496 token_bytes=557927944 weight_movement_bytes=220200960 total_nvlink_bytes=778128904 weight_to_token_ratio=0.394676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679308922e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280297472 dispatch_meta_bytes=229024 combine_payload_bytes=280297472 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280526496 ingress_bytes=500498432 link_bytes=500498432 token_bytes=560823968 weight_movement_bytes=220200960 total_nvlink_bytes=781024928 weight_to_token_ratio=0.392638 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.704490911e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=230312 combine_payload_bytes=278992896 prefetch_bytes=0 reduce_bytes=0 egress_bytes=279223208 ingress_bytes=278992896 link_bytes=279223208 token_bytes=558216104 weight_movement_bytes=0 total_nvlink_bytes=558216104 weight_to_token_ratio=0.000000 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=4.856150830e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229192 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279336776 ingress_bytes=499308544 link_bytes=499308544 token_bytes=558444360 weight_movement_bytes=220200960 total_nvlink_bytes=778645320 weight_to_token_ratio=0.394311 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683796802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44424 payload_rows_remote=38788 dup_rows=21112 dup_rows_remote=18494 dedup_ratio=0.322144 router_tokens_per_expert=[526, 499, 496, 485, 538, 494, 544, 490, 506, 510, 544, 519, 473, 461, 509, 541, 462, 493, 541, 488, 520, 569, 521, 492, 509, 473, 537, 561, 504, 475, 506, 543, 550, 514, 525, 526, 476, 524, 549, 496, 515, 522, 522, 511, 517, 511, 503, 526, 525, 486, 524, 504, 511, 503, 540, 497, 536, 488, 481, 492, 479, 545, 513, 528] payload_rows_per_dst=[5564, 5528, 5506, 5550, 5592, 5636, 5490, 5558] dup_rows_per_dst=[2580, 2598, 2666, 2666, 2728, 2618, 2690, 2566] combine_rows_per_src=[5564, 5528, 5506, 5550, 5592, 5636, 5490, 5558] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280297472 dispatch_meta_bytes=229024 combine_payload_bytes=280297472 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280526496 ingress_bytes=500498432 link_bytes=500498432 token_bytes=560823968 weight_movement_bytes=220200960 total_nvlink_bytes=781024928 weight_to_token_ratio=0.392638 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.704490911e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=228872 combine_payload_bytes=278849536 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279078408 ingress_bytes=499050496 link_bytes=499050496 token_bytes=557927944 weight_movement_bytes=220200960 total_nvlink_bytes=778128904 weight_to_token_ratio=0.394676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679308922e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278032384 dispatch_meta_bytes=229128 combine_payload_bytes=278032384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278261512 ingress_bytes=388132864 link_bytes=388132864 token_bytes=556293896 weight_movement_bytes=110100480 total_nvlink_bytes=666394376 weight_to_token_ratio=0.197918 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750268874e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280053760 dispatch_meta_bytes=228896 combine_payload_bytes=280053760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280282656 ingress_bytes=390154240 link_bytes=390154240 token_bytes=560336416 weight_movement_bytes=110100480 total_nvlink_bytes=670436896 weight_to_token_ratio=0.196490 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785423927e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44474 payload_rows_remote=38882 dup_rows=21062 dup_rows_remote=18406 dedup_ratio=0.321381 router_tokens_per_expert=[503, 535, 532, 540, 489, 542, 488, 551, 539, 504, 546, 475, 474, 520, 549, 493, 538, 463, 515, 536, 513, 532, 509, 518, 487, 477, 497, 530, 488, 547, 510, 514, 526, 514, 472, 522, 514, 513, 517, 495, 496, 508, 553, 457, 522, 520, 519, 498, 526, 536, 551, 487, 484, 462, 505, 541, 533, 533, 461, 506, 517, 519, 504, 503] payload_rows_per_dst=[5564, 5584, 5592, 5576, 5476, 5504, 5570, 5608] dup_rows_per_dst=[2796, 2616, 2656, 2524, 2670, 2642, 2614, 2544] combine_rows_per_src=[5564, 5584, 5592, 5576, 5476, 5504, 5570, 5608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38892 dup_rows=21108 dup_rows_remote=18516 dedup_ratio=0.322083 router_tokens_per_expert=[521, 526, 500, 494, 485, 493, 536, 509, 476, 533, 542, 490, 506, 552, 505, 550, 485, 508, 491, 514, 556, 505, 493, 524, 487, 520, 504, 510, 496, 532, 457, 496, 475, 483, 515, 496, 555, 525, 511, 532, 494, 510, 554, 495, 537, 470, 463, 501, 537, 527, 542, 506, 513, 555, 504, 525, 525, 525, 487, 534, 514, 527, 510, 525] payload_rows_per_dst=[5536, 5654, 5536, 5494, 5560, 5444, 5632, 5572] dup_rows_per_dst=[2592, 2712, 2616, 2510, 2644, 2604, 2786, 2644] combine_rows_per_src=[5536, 5654, 5536, 5494, 5560, 5444, 5632, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44696 payload_rows_remote=39128 dup_rows=20840 dup_rows_remote=18192 dedup_ratio=0.317993 router_tokens_per_expert=[509, 535, 536, 487, 517, 510, 468, 532, 517, 543, 507, 499, 469, 537, 534, 502, 505, 539, 482, 474, 514, 553, 518, 519, 505, 538, 513, 481, 496, 497, 547, 533, 524, 517, 529, 504, 511, 501, 518, 486, 501, 499, 529, 501, 497, 521, 514, 531, 555, 517, 497, 482, 508, 498, 503, 515, 505, 513, 523, 496, 518, 532, 509, 498] payload_rows_per_dst=[5652, 5568, 5622, 5616, 5564, 5552, 5580, 5542] dup_rows_per_dst=[2536, 2648, 2586, 2604, 2616, 2634, 2570, 2646] combine_rows_per_src=[5652, 5568, 5622, 5616, 5564, 5552, 5580, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=39022 dup_rows=21018 dup_rows_remote=18462 dedup_ratio=0.320709 router_tokens_per_expert=[540, 527, 540, 488, 537, 534, 499, 499, 503, 510, 505, 519, 519, 537, 508, 544, 501, 547, 520, 516, 525, 498, 546, 511, 524, 484, 494, 497, 490, 522, 524, 516, 508, 517, 536, 507, 476, 508, 481, 494, 510, 525, 498, 512, 518, 506, 507, 533, 499, 486, 488, 476, 493, 533, 558, 493, 522, 510, 518, 519, 512, 506, 496, 499] payload_rows_per_dst=[5680, 5600, 5620, 5514, 5492, 5592, 5496, 5524] dup_rows_per_dst=[2648, 2690, 2708, 2588, 2562, 2626, 2556, 2640] combine_rows_per_src=[5680, 5600, 5620, 5514, 5492, 5592, 5496, 5524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=229152 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278935328 ingress_bytes=388806656 link_bytes=388806656 token_bytes=557641504 weight_movement_bytes=110100480 total_nvlink_bytes=667741984 weight_to_token_ratio=0.197440 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761987225e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44630 payload_rows_remote=39050 dup_rows=20906 dup_rows_remote=18234 dedup_ratio=0.319000 router_tokens_per_expert=[487, 522, 545, 559, 539, 474, 517, 510, 495, 540, 548, 492, 508, 531, 511, 496, 511, 476, 527, 495, 526, 506, 477, 461, 491, 483, 525, 474, 531, 497, 506, 520, 516, 517, 515, 536, 532, 482, 509, 519, 529, 543, 492, 536, 537, 518, 494, 515, 481, 505, 540, 525, 535, 527, 535, 527, 492, 519, 479, 525, 502, 496, 502, 508] payload_rows_per_dst=[5596, 5584, 5484, 5546, 5580, 5654, 5632, 5554] dup_rows_per_dst=[2710, 2658, 2474, 2508, 2672, 2674, 2718, 2492] combine_rows_per_src=[5596, 5584, 5484, 5546, 5580, 5654, 5632, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=280053760 dispatch_meta_bytes=228896 combine_payload_bytes=280053760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280282656 ingress_bytes=390154240 link_bytes=390154240 token_bytes=560336416 weight_movement_bytes=110100480 total_nvlink_bytes=670436896 weight_to_token_ratio=0.196490 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785423927e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=229632 combine_payload_bytes=278777856 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279007488 ingress_bytes=388878336 link_bytes=388878336 token_bytes=557785344 weight_movement_bytes=110100480 total_nvlink_bytes=667885824 weight_to_token_ratio=0.197389 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763233858e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=229840 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279108048 ingress_bytes=719280128 link_bytes=719280128 token_bytes=557986256 weight_movement_bytes=440401920 total_nvlink_bytes=998388176 weight_to_token_ratio=0.789270 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250946444e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278032384 dispatch_meta_bytes=229128 combine_payload_bytes=278032384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278261512 ingress_bytes=388132864 link_bytes=388132864 token_bytes=556293896 weight_movement_bytes=110100480 total_nvlink_bytes=666394376 weight_to_token_ratio=0.197918 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750268874e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44338 payload_rows_remote=38754 dup_rows=21198 dup_rows_remote=18466 dedup_ratio=0.323456 router_tokens_per_expert=[507, 469, 494, 538, 490, 509, 509, 505, 500, 559, 501, 488, 491, 481, 501, 502, 531, 485, 487, 485, 521, 512, 509, 503, 524, 506, 512, 536, 534, 494, 525, 505, 519, 542, 526, 530, 525, 539, 507, 518, 518, 530, 525, 489, 503, 503, 481, 540, 527, 529, 519, 521, 505, 517, 480, 505, 553, 547, 495, 543, 525, 504, 497, 493] payload_rows_per_dst=[5480, 5514, 5512, 5600, 5532, 5526, 5590, 5584] dup_rows_per_dst=[2562, 2532, 2554, 2582, 2880, 2650, 2706, 2732] combine_rows_per_src=[5480, 5514, 5512, 5600, 5532, 5526, 5590, 5584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44306 payload_rows_remote=38684 dup_rows=21230 dup_rows_remote=18404 dedup_ratio=0.323944 router_tokens_per_expert=[479, 533, 500, 536, 483, 522, 518, 477, 465, 494, 508, 515, 530, 528, 532, 494, 520, 507, 485, 492, 508, 528, 464, 488, 461, 494, 508, 537, 495, 533, 516, 507, 550, 530, 501, 493, 486, 547, 544, 529, 525, 545, 515, 580, 490, 530, 526, 513, 498, 497, 511, 513, 531, 497, 573, 511, 488, 486, 531, 493, 528, 484, 526, 540] payload_rows_per_dst=[5546, 5560, 5460, 5458, 5554, 5622, 5580, 5526] dup_rows_per_dst=[2550, 2572, 2524, 2644, 2806, 2826, 2682, 2626] combine_rows_per_src=[5546, 5560, 5460, 5458, 5554, 5622, 5580, 5526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280469504 dispatch_meta_bytes=229280 combine_payload_bytes=280469504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280698784 ingress_bytes=390569984 link_bytes=390569984 token_bytes=561168288 weight_movement_bytes=110100480 total_nvlink_bytes=671268768 weight_to_token_ratio=0.196199 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.792654399e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44630 payload_rows_remote=39050 dup_rows=20906 dup_rows_remote=18234 dedup_ratio=0.319000 router_tokens_per_expert=[487, 522, 545, 559, 539, 474, 517, 510, 495, 540, 548, 492, 508, 531, 511, 496, 511, 476, 527, 495, 526, 506, 477, 461, 491, 483, 525, 474, 531, 497, 506, 520, 516, 517, 515, 536, 532, 482, 509, 519, 529, 543, 492, 536, 537, 518, 494, 515, 481, 505, 540, 525, 535, 527, 535, 527, 492, 519, 479, 525, 502, 496, 502, 508] payload_rows_per_dst=[5596, 5584, 5484, 5546, 5580, 5654, 5632, 5554] dup_rows_per_dst=[2710, 2658, 2474, 2508, 2672, 2674, 2718, 2492] combine_rows_per_src=[5596, 5584, 5484, 5546, 5580, 5654, 5632, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44518 payload_rows_remote=39022 dup_rows=21018 dup_rows_remote=18462 dedup_ratio=0.320709 router_tokens_per_expert=[540, 527, 540, 488, 537, 534, 499, 499, 503, 510, 505, 519, 519, 537, 508, 544, 501, 547, 520, 516, 525, 498, 546, 511, 524, 484, 494, 497, 490, 522, 524, 516, 508, 517, 536, 507, 476, 508, 481, 494, 510, 525, 498, 512, 518, 506, 507, 533, 499, 486, 488, 476, 493, 533, 558, 493, 522, 510, 518, 519, 512, 506, 496, 499] payload_rows_per_dst=[5680, 5600, 5620, 5514, 5492, 5592, 5496, 5524] dup_rows_per_dst=[2648, 2690, 2708, 2588, 2562, 2626, 2556, 2640] combine_rows_per_src=[5680, 5600, 5620, 5514, 5492, 5592, 5496, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44696 payload_rows_remote=39128 dup_rows=20840 dup_rows_remote=18192 dedup_ratio=0.317993 router_tokens_per_expert=[509, 535, 536, 487, 517, 510, 468, 532, 517, 543, 507, 499, 469, 537, 534, 502, 505, 539, 482, 474, 514, 553, 518, 519, 505, 538, 513, 481, 496, 497, 547, 533, 524, 517, 529, 504, 511, 501, 518, 486, 501, 499, 529, 501, 497, 521, 514, 531, 555, 517, 497, 482, 508, 498, 503, 515, 505, 513, 523, 496, 518, 532, 509, 498] payload_rows_per_dst=[5652, 5568, 5622, 5616, 5564, 5552, 5580, 5542] dup_rows_per_dst=[2536, 2648, 2586, 2604, 2616, 2634, 2570, 2646] combine_rows_per_src=[5652, 5568, 5622, 5616, 5564, 5552, 5580, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38940 dup_rows=20976 dup_rows_remote=18218 dedup_ratio=0.320068 router_tokens_per_expert=[512, 506, 509, 495, 483, 499, 498, 508, 500, 476, 507, 542, 526, 492, 510, 534, 508, 524, 497, 512, 500, 499, 505, 523, 503, 493, 564, 535, 529, 537, 501, 527, 530, 486, 528, 514, 522, 509, 510, 539, 521, 490, 511, 473, 539, 523, 525, 502, 486, 562, 487, 541, 514, 505, 523, 512, 509, 499, 497, 515, 511, 557, 455, 519] payload_rows_per_dst=[5538, 5658, 5498, 5620, 5606, 5542, 5558, 5540] dup_rows_per_dst=[2482, 2516, 2638, 2758, 2670, 2626, 2702, 2584] combine_rows_per_src=[5538, 5658, 5498, 5620, 5606, 5542, 5558, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279910400 dispatch_meta_bytes=229136 combine_payload_bytes=279910400 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280139536 ingress_bytes=610211840 link_bytes=610211840 token_bytes=560049936 weight_movement_bytes=330301440 total_nvlink_bytes=890351376 weight_to_token_ratio=0.589771 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061258752e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44314 payload_rows_remote=38696 dup_rows=21222 dup_rows_remote=18442 dedup_ratio=0.323822 router_tokens_per_expert=[511, 530, 515, 512, 496, 516, 519, 497, 513, 472, 499, 526, 501, 521, 518, 498, 572, 495, 472, 474, 498, 544, 531, 516, 518, 488, 547, 490, 503, 514, 501, 515, 522, 531, 497, 499, 480, 512, 500, 523, 529, 543, 521, 494, 518, 480, 523, 479, 537, 516, 537, 501, 473, 518, 502, 520, 501, 531, 531, 541, 525, 516, 517, 529] payload_rows_per_dst=[5618, 5600, 5744, 5556, 5460, 5310, 5416, 5610] dup_rows_per_dst=[2780, 2782, 2994, 2614, 2370, 2330, 2586, 2766] combine_rows_per_src=[5618, 5600, 5744, 5556, 5460, 5310, 5416, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277788672 dispatch_meta_bytes=228880 combine_payload_bytes=277788672 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278017552 ingress_bytes=497989632 link_bytes=497989632 token_bytes=555806224 weight_movement_bytes=220200960 total_nvlink_bytes=776007184 weight_to_token_ratio=0.396183 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.660858753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44620 payload_rows_remote=39066 dup_rows=20916 dup_rows_remote=18250 dedup_ratio=0.319153 router_tokens_per_expert=[520, 505, 536, 516, 486, 535, 522, 483, 536, 519, 502, 496, 486, 511, 526, 497, 546, 453, 526, 528, 542, 497, 526, 492, 526, 516, 471, 492, 526, 551, 497, 519, 482, 510, 482, 527, 518, 553, 521, 543, 480, 529, 492, 511, 532, 522, 502, 513, 532, 500, 495, 487, 502, 502, 552, 516, 531, 500, 539, 518, 484, 486, 512, 511] payload_rows_per_dst=[5622, 5586, 5554, 5596, 5624, 5500, 5564, 5574] dup_rows_per_dst=[2584, 2560, 2666, 2600, 2648, 2662, 2608, 2588] combine_rows_per_src=[5622, 5586, 5554, 5596, 5624, 5500, 5564, 5574] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279709696 dispatch_meta_bytes=229936 combine_payload_bytes=279709696 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279939632 ingress_bytes=499910656 link_bytes=499910656 token_bytes=559649328 weight_movement_bytes=220200960 total_nvlink_bytes=779850288 weight_to_token_ratio=0.393462 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694268520e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279910400 dispatch_meta_bytes=229136 combine_payload_bytes=279910400 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280139536 ingress_bytes=610211840 link_bytes=610211840 token_bytes=560049936 weight_movement_bytes=330301440 total_nvlink_bytes=890351376 weight_to_token_ratio=0.589771 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061258752e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280469504 dispatch_meta_bytes=229280 combine_payload_bytes=280469504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280698784 ingress_bytes=390569984 link_bytes=390569984 token_bytes=561168288 weight_movement_bytes=110100480 total_nvlink_bytes=671268768 weight_to_token_ratio=0.196199 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.792654399e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44306 payload_rows_remote=38684 dup_rows=21230 dup_rows_remote=18404 dedup_ratio=0.323944 router_tokens_per_expert=[479, 533, 500, 536, 483, 522, 518, 477, 465, 494, 508, 515, 530, 528, 532, 494, 520, 507, 485, 492, 508, 528, 464, 488, 461, 494, 508, 537, 495, 533, 516, 507, 550, 530, 501, 493, 486, 547, 544, 529, 525, 545, 515, 580, 490, 530, 526, 513, 498, 497, 511, 513, 531, 497, 573, 511, 488, 486, 531, 493, 528, 484, 526, 540] payload_rows_per_dst=[5546, 5560, 5460, 5458, 5554, 5622, 5580, 5526] dup_rows_per_dst=[2550, 2572, 2524, 2644, 2806, 2826, 2682, 2626] combine_rows_per_src=[5546, 5560, 5460, 5458, 5554, 5622, 5580, 5526] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277788672 dispatch_meta_bytes=228880 combine_payload_bytes=277788672 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278017552 ingress_bytes=497989632 link_bytes=497989632 token_bytes=555806224 weight_movement_bytes=220200960 total_nvlink_bytes=776007184 weight_to_token_ratio=0.396183 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.660858753e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277286912 dispatch_meta_bytes=228352 combine_payload_bytes=277286912 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277515264 ingress_bytes=497487872 link_bytes=497487872 token_bytes=554802176 weight_movement_bytes=220200960 total_nvlink_bytes=775003136 weight_to_token_ratio=0.396900 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652132321e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=228632 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279350552 ingress_bytes=499322880 link_bytes=499322880 token_bytes=558472472 weight_movement_bytes=220200960 total_nvlink_bytes=778673432 weight_to_token_ratio=0.394292 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684046128e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=39038 dup_rows=21004 dup_rows_remote=18380 dedup_ratio=0.320496 router_tokens_per_expert=[504, 511, 513, 532, 499, 529, 496, 495, 510, 492, 501, 483, 535, 514, 511, 481, 520, 554, 496, 488, 541, 465, 477, 518, 535, 494, 474, 525, 566, 520, 514, 536, 506, 504, 513, 479, 489, 537, 543, 515, 592, 503, 481, 532, 507, 533, 513, 525, 487, 492, 534, 528, 517, 535, 503, 486, 522, 485, 521, 512, 498, 499, 522, 526] payload_rows_per_dst=[5558, 5462, 5494, 5622, 5596, 5718, 5550, 5532] dup_rows_per_dst=[2600, 2592, 2624, 2706, 2576, 2654, 2614, 2638] combine_rows_per_src=[5558, 5462, 5494, 5622, 5596, 5718, 5550, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44314 payload_rows_remote=38696 dup_rows=21222 dup_rows_remote=18442 dedup_ratio=0.323822 router_tokens_per_expert=[511, 530, 515, 512, 496, 516, 519, 497, 513, 472, 499, 526, 501, 521, 518, 498, 572, 495, 472, 474, 498, 544, 531, 516, 518, 488, 547, 490, 503, 514, 501, 515, 522, 531, 497, 499, 480, 512, 500, 523, 529, 543, 521, 494, 518, 480, 523, 479, 537, 516, 537, 501, 473, 518, 502, 520, 501, 531, 531, 541, 525, 516, 517, 529] payload_rows_per_dst=[5618, 5600, 5744, 5556, 5460, 5310, 5416, 5610] dup_rows_per_dst=[2780, 2782, 2994, 2614, 2370, 2330, 2586, 2766] combine_rows_per_src=[5618, 5600, 5744, 5556, 5460, 5310, 5416, 5610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38888 dup_rows=21086 dup_rows_remote=18594 dedup_ratio=0.321747 router_tokens_per_expert=[466, 508, 521, 520, 535, 529, 543, 534, 471, 524, 514, 524, 503, 526, 492, 473, 513, 499, 490, 535, 521, 515, 491, 555, 511, 509, 532, 511, 504, 471, 532, 486, 529, 518, 494, 473, 504, 507, 513, 534, 511, 504, 536, 549, 539, 514, 531, 514, 490, 460, 475, 515, 501, 525, 541, 505, 511, 513, 512, 527, 510, 559, 482, 514] payload_rows_per_dst=[5616, 5562, 5578, 5530, 5478, 5670, 5468, 5548] dup_rows_per_dst=[2696, 2492, 2660, 2582, 2666, 2726, 2556, 2708] combine_rows_per_src=[5616, 5562, 5578, 5530, 5478, 5670, 5468, 5548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279824384 dispatch_meta_bytes=229672 combine_payload_bytes=279824384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280054056 ingress_bytes=389924864 link_bytes=389924864 token_bytes=559878440 weight_movement_bytes=110100480 total_nvlink_bytes=669978920 weight_to_token_ratio=0.196651 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781434701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38898 dup_rows=21138 dup_rows_remote=18606 dedup_ratio=0.322540 router_tokens_per_expert=[493, 532, 482, 509, 497, 496, 520, 544, 511, 541, 490, 491, 500, 522, 526, 514, 505, 514, 494, 528, 523, 509, 525, 520, 523, 503, 514, 506, 550, 498, 535, 492, 485, 514, 480, 520, 492, 518, 505, 502, 510, 520, 509, 539, 492, 533, 507, 515, 502, 532, 526, 497, 523, 513, 504, 525, 514, 497, 540, 500, 527, 500, 514, 506] payload_rows_per_dst=[5518, 5630, 5510, 5590, 5500, 5560, 5538, 5552] dup_rows_per_dst=[2628, 2560, 2726, 2652, 2532, 2690, 2706, 2644] combine_rows_per_src=[5518, 5630, 5510, 5590, 5500, 5560, 5538, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=228632 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279350552 ingress_bytes=499322880 link_bytes=499322880 token_bytes=558472472 weight_movement_bytes=220200960 total_nvlink_bytes=778673432 weight_to_token_ratio=0.394292 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684046128e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277372928 dispatch_meta_bytes=228552 combine_payload_bytes=277372928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277601480 ingress_bytes=387473408 link_bytes=387473408 token_bytes=554974408 weight_movement_bytes=110100480 total_nvlink_bytes=665074888 weight_to_token_ratio=0.198388 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.738799849e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44644 payload_rows_remote=38972 dup_rows=20892 dup_rows_remote=18290 dedup_ratio=0.318787 router_tokens_per_expert=[508, 499, 499, 519, 496, 469, 497, 525, 471, 485, 482, 494, 488, 510, 504, 501, 538, 524, 501, 528, 503, 562, 527, 483, 497, 512, 506, 517, 558, 530, 533, 539, 536, 518, 503, 529, 549, 523, 512, 525, 487, 525, 512, 489, 467, 508, 517, 521, 527, 535, 516, 490, 507, 492, 554, 516, 499, 505, 537, 502, 538, 511, 476, 537] payload_rows_per_dst=[5508, 5498, 5668, 5610, 5582, 5490, 5672, 5616] dup_rows_per_dst=[2516, 2372, 2664, 2774, 2808, 2562, 2602, 2594] combine_rows_per_src=[5508, 5498, 5668, 5610, 5582, 5490, 5672, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=229264 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280254352 ingress_bytes=500226048 link_bytes=500226048 token_bytes=560279440 weight_movement_bytes=220200960 total_nvlink_bytes=780480400 weight_to_token_ratio=0.393020 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.699753705e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277286912 dispatch_meta_bytes=228352 combine_payload_bytes=277286912 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277515264 ingress_bytes=497487872 link_bytes=497487872 token_bytes=554802176 weight_movement_bytes=220200960 total_nvlink_bytes=775003136 weight_to_token_ratio=0.396900 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652132321e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38858 dup_rows=21130 dup_rows_remote=18434 dedup_ratio=0.322418 router_tokens_per_expert=[504, 532, 507, 478, 505, 485, 547, 534, 530, 482, 522, 485, 484, 525, 490, 514, 498, 536, 504, 514, 489, 515, 469, 549, 542, 517, 532, 541, 484, 527, 502, 522, 500, 517, 455, 500, 522, 538, 508, 486, 528, 535, 479, 530, 536, 560, 535, 533, 510, 537, 509, 487, 527, 500, 517, 479, 493, 497, 488, 504, 552, 486, 538, 517] payload_rows_per_dst=[5474, 5474, 5572, 5358, 5606, 5660, 5714, 5548] dup_rows_per_dst=[2406, 2590, 2688, 2428, 2614, 2718, 2990, 2696] combine_rows_per_src=[5474, 5474, 5572, 5358, 5606, 5660, 5714, 5548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229928 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278979112 ingress_bytes=388849664 link_bytes=388849664 token_bytes=557728296 weight_movement_bytes=110100480 total_nvlink_bytes=667828776 weight_to_token_ratio=0.197409 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762735205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39030 dup_rows=20988 dup_rows_remote=18416 dedup_ratio=0.320251 router_tokens_per_expert=[538, 510, 509, 529, 522, 529, 499, 501, 502, 515, 501, 522, 533, 516, 512, 538, 512, 527, 521, 524, 490, 470, 534, 531, 522, 525, 500, 538, 510, 517, 538, 497, 502, 556, 497, 528, 492, 544, 509, 556, 548, 479, 476, 481, 537, 494, 499, 531, 531, 466, 483, 527, 513, 520, 482, 513, 530, 482, 485, 513, 481, 502, 474, 505] payload_rows_per_dst=[5586, 5634, 5588, 5558, 5670, 5518, 5514, 5480] dup_rows_per_dst=[2688, 2644, 2630, 2736, 2698, 2572, 2556, 2464] combine_rows_per_src=[5586, 5634, 5588, 5558, 5670, 5518, 5514, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38898 dup_rows=21138 dup_rows_remote=18606 dedup_ratio=0.322540 router_tokens_per_expert=[493, 532, 482, 509, 497, 496, 520, 544, 511, 541, 490, 491, 500, 522, 526, 514, 505, 514, 494, 528, 523, 509, 525, 520, 523, 503, 514, 506, 550, 498, 535, 492, 485, 514, 480, 520, 492, 518, 505, 502, 510, 520, 509, 539, 492, 533, 507, 515, 502, 532, 526, 497, 523, 513, 504, 525, 514, 497, 540, 500, 527, 500, 514, 506] payload_rows_per_dst=[5518, 5630, 5510, 5590, 5500, 5560, 5538, 5552] dup_rows_per_dst=[2628, 2560, 2726, 2652, 2532, 2690, 2706, 2644] combine_rows_per_src=[5518, 5630, 5510, 5590, 5500, 5560, 5538, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38888 dup_rows=21086 dup_rows_remote=18594 dedup_ratio=0.321747 router_tokens_per_expert=[466, 508, 521, 520, 535, 529, 543, 534, 471, 524, 514, 524, 503, 526, 492, 473, 513, 499, 490, 535, 521, 515, 491, 555, 511, 509, 532, 511, 504, 471, 532, 486, 529, 518, 494, 473, 504, 507, 513, 534, 511, 504, 536, 549, 539, 514, 531, 514, 490, 460, 475, 515, 501, 525, 541, 505, 511, 513, 512, 527, 510, 559, 482, 514] payload_rows_per_dst=[5616, 5562, 5578, 5530, 5478, 5670, 5468, 5548] dup_rows_per_dst=[2696, 2492, 2660, 2582, 2666, 2726, 2556, 2708] combine_rows_per_src=[5616, 5562, 5578, 5530, 5478, 5670, 5468, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44644 payload_rows_remote=38972 dup_rows=20892 dup_rows_remote=18290 dedup_ratio=0.318787 router_tokens_per_expert=[508, 499, 499, 519, 496, 469, 497, 525, 471, 485, 482, 494, 488, 510, 504, 501, 538, 524, 501, 528, 503, 562, 527, 483, 497, 512, 506, 517, 558, 530, 533, 539, 536, 518, 503, 529, 549, 523, 512, 525, 487, 525, 512, 489, 467, 508, 517, 521, 527, 535, 516, 490, 507, 492, 554, 516, 499, 505, 537, 502, 538, 511, 476, 537] payload_rows_per_dst=[5508, 5498, 5668, 5610, 5582, 5490, 5672, 5616] dup_rows_per_dst=[2516, 2372, 2664, 2774, 2808, 2562, 2602, 2594] combine_rows_per_src=[5508, 5498, 5668, 5610, 5582, 5490, 5672, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=229168 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278763312 ingress_bytes=388634624 link_bytes=388634624 token_bytes=557297456 weight_movement_bytes=110100480 total_nvlink_bytes=667397936 weight_to_token_ratio=0.197561 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758995305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44322 payload_rows_remote=38688 dup_rows=21214 dup_rows_remote=18588 dedup_ratio=0.323700 router_tokens_per_expert=[497, 525, 505, 525, 504, 504, 520, 531, 486, 543, 482, 509, 516, 507, 492, 520, 484, 477, 537, 526, 570, 508, 532, 535, 533, 537, 497, 520, 514, 486, 551, 492, 528, 515, 500, 495, 491, 483, 534, 545, 525, 523, 483, 532, 497, 531, 508, 533, 464, 515, 503, 491, 454, 538, 500, 506, 517, 545, 521, 477, 539, 499, 490, 521] payload_rows_per_dst=[5522, 5472, 5566, 5634, 5500, 5610, 5436, 5582] dup_rows_per_dst=[2700, 2638, 2772, 2626, 2682, 2654, 2506, 2636] combine_rows_per_src=[5522, 5472, 5566, 5634, 5500, 5610, 5436, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38784 dup_rows=21108 dup_rows_remote=18398 dedup_ratio=0.322083 router_tokens_per_expert=[549, 537, 511, 499, 518, 534, 524, 505, 503, 543, 529, 496, 462, 519, 535, 533, 509, 518, 513, 522, 524, 499, 486, 527, 530, 507, 513, 502, 543, 515, 484, 489, 477, 492, 523, 529, 492, 515, 495, 493, 506, 492, 480, 476, 516, 508, 534, 504, 522, 499, 513, 561, 494, 511, 489, 484, 523, 514, 532, 536, 502, 536, 494, 548] payload_rows_per_dst=[5644, 5788, 5268, 5570, 5432, 5488, 5516, 5722] dup_rows_per_dst=[2710, 2986, 2394, 2596, 2600, 2544, 2630, 2648] combine_rows_per_src=[5644, 5788, 5268, 5570, 5432, 5488, 5516, 5722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229928 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278979112 ingress_bytes=388849664 link_bytes=388849664 token_bytes=557728296 weight_movement_bytes=110100480 total_nvlink_bytes=667828776 weight_to_token_ratio=0.197409 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762735205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278820864 dispatch_meta_bytes=230016 combine_payload_bytes=278820864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279050880 ingress_bytes=388921344 link_bytes=388921344 token_bytes=557871744 weight_movement_bytes=110100480 total_nvlink_bytes=667972224 weight_to_token_ratio=0.197358 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763981838e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44474 payload_rows_remote=38882 dup_rows=21062 dup_rows_remote=18406 dedup_ratio=0.321381 router_tokens_per_expert=[503, 535, 532, 540, 489, 542, 488, 551, 539, 504, 546, 475, 474, 520, 549, 493, 538, 463, 515, 536, 513, 532, 509, 518, 487, 477, 497, 530, 488, 547, 510, 514, 526, 514, 472, 522, 514, 513, 517, 495, 496, 508, 553, 457, 522, 520, 519, 498, 526, 536, 551, 487, 484, 462, 505, 541, 533, 533, 461, 506, 517, 519, 504, 503] payload_rows_per_dst=[5564, 5584, 5592, 5576, 5476, 5504, 5570, 5608] dup_rows_per_dst=[2796, 2616, 2656, 2524, 2670, 2642, 2614, 2544] combine_rows_per_src=[5564, 5584, 5592, 5576, 5476, 5504, 5570, 5608] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279351296 dispatch_meta_bytes=229048 combine_payload_bytes=279351296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279580344 ingress_bytes=389451776 link_bytes=389451776 token_bytes=558931640 weight_movement_bytes=110100480 total_nvlink_bytes=669032120 weight_to_token_ratio=0.196984 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773206923e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278534144 dispatch_meta_bytes=229168 combine_payload_bytes=278534144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278763312 ingress_bytes=388634624 link_bytes=388634624 token_bytes=557297456 weight_movement_bytes=110100480 total_nvlink_bytes=667397936 weight_to_token_ratio=0.197561 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758995305e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=229784 combine_payload_bytes=279767040 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279996824 ingress_bytes=389867520 link_bytes=389867520 token_bytes=559763864 weight_movement_bytes=110100480 total_nvlink_bytes=669864344 weight_to_token_ratio=0.196691 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780437395e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39030 dup_rows=20988 dup_rows_remote=18416 dedup_ratio=0.320251 router_tokens_per_expert=[538, 510, 509, 529, 522, 529, 499, 501, 502, 515, 501, 522, 533, 516, 512, 538, 512, 527, 521, 524, 490, 470, 534, 531, 522, 525, 500, 538, 510, 517, 538, 497, 502, 556, 497, 528, 492, 544, 509, 556, 548, 479, 476, 481, 537, 494, 499, 531, 531, 466, 483, 527, 513, 520, 482, 513, 530, 482, 485, 513, 481, 502, 474, 505] payload_rows_per_dst=[5586, 5634, 5588, 5558, 5670, 5518, 5514, 5480] dup_rows_per_dst=[2688, 2644, 2630, 2736, 2698, 2572, 2556, 2464] combine_rows_per_src=[5586, 5634, 5588, 5558, 5670, 5518, 5514, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277315584 dispatch_meta_bytes=229104 combine_payload_bytes=277315584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277544688 ingress_bytes=497516544 link_bytes=497516544 token_bytes=554860272 weight_movement_bytes=220200960 total_nvlink_bytes=775061232 weight_to_token_ratio=0.396858 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652630974e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38848 dup_rows=21150 dup_rows_remote=18436 dedup_ratio=0.322723 router_tokens_per_expert=[524, 544, 525, 514, 509, 523, 500, 523, 517, 507, 509, 518, 513, 527, 490, 517, 523, 490, 564, 498, 523, 544, 478, 506, 479, 492, 534, 541, 525, 543, 538, 494, 530, 455, 508, 501, 474, 549, 531, 536, 524, 527, 471, 527, 510, 505, 476, 497, 507, 526, 520, 536, 492, 488, 492, 505, 493, 510, 508, 481, 551, 478, 490, 538] payload_rows_per_dst=[5624, 5564, 5538, 5658, 5516, 5456, 5532, 5498] dup_rows_per_dst=[2700, 2632, 2714, 2634, 2652, 2618, 2600, 2600] combine_rows_per_src=[5624, 5564, 5538, 5658, 5516, 5456, 5532, 5498] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38784 dup_rows=21108 dup_rows_remote=18398 dedup_ratio=0.322083 router_tokens_per_expert=[549, 537, 511, 499, 518, 534, 524, 505, 503, 543, 529, 496, 462, 519, 535, 533, 509, 518, 513, 522, 524, 499, 486, 527, 530, 507, 513, 502, 543, 515, 484, 489, 477, 492, 523, 529, 492, 515, 495, 493, 506, 492, 480, 476, 516, 508, 534, 504, 522, 499, 513, 561, 494, 511, 489, 484, 523, 514, 532, 536, 502, 536, 494, 548] payload_rows_per_dst=[5644, 5788, 5268, 5570, 5432, 5488, 5516, 5722] dup_rows_per_dst=[2710, 2986, 2394, 2596, 2600, 2544, 2630, 2648] combine_rows_per_src=[5644, 5788, 5268, 5570, 5432, 5488, 5516, 5722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278003712 dispatch_meta_bytes=228728 combine_payload_bytes=278003712 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278232440 ingress_bytes=608305152 link_bytes=608305152 token_bytes=556236152 weight_movement_bytes=330301440 total_nvlink_bytes=886537592 weight_to_token_ratio=0.593815 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057942708e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=229152 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278935328 ingress_bytes=388806656 link_bytes=388806656 token_bytes=557641504 weight_movement_bytes=110100480 total_nvlink_bytes=667741984 weight_to_token_ratio=0.197440 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761987225e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=229784 combine_payload_bytes=279767040 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279996824 ingress_bytes=389867520 link_bytes=389867520 token_bytes=559763864 weight_movement_bytes=110100480 total_nvlink_bytes=669864344 weight_to_token_ratio=0.196691 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780437395e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38814 dup_rows=21034 dup_rows_remote=18194 dedup_ratio=0.320953 router_tokens_per_expert=[517, 512, 488, 531, 533, 476, 519, 536, 529, 499, 531, 522, 485, 520, 465, 510, 530, 539, 495, 516, 491, 490, 488, 480, 524, 484, 517, 555, 517, 516, 533, 532, 504, 498, 524, 501, 553, 519, 499, 492, 527, 523, 526, 528, 496, 500, 512, 517, 534, 543, 485, 508, 512, 506, 522, 534, 480, 482, 490, 484, 512, 521, 530, 526] payload_rows_per_dst=[5560, 5516, 5488, 5566, 5660, 5418, 5606, 5688] dup_rows_per_dst=[2628, 2606, 2570, 2736, 2610, 2318, 2726, 2840] combine_rows_per_src=[5560, 5516, 5488, 5566, 5660, 5418, 5606, 5688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44306 payload_rows_remote=38768 dup_rows=21230 dup_rows_remote=18700 dedup_ratio=0.323944 router_tokens_per_expert=[484, 512, 550, 544, 497, 515, 532, 499, 534, 481, 484, 481, 526, 490, 521, 517, 535, 507, 513, 523, 477, 540, 493, 474, 541, 521, 538, 533, 518, 503, 534, 523, 500, 553, 527, 483, 530, 524, 528, 529, 491, 523, 519, 501, 522, 512, 476, 498, 536, 468, 529, 516, 502, 542, 496, 498, 508, 510, 519, 519, 468, 526, 473, 502] payload_rows_per_dst=[5536, 5538, 5536, 5622, 5550, 5494, 5546, 5484] dup_rows_per_dst=[2730, 2530, 2588, 2800, 2798, 2590, 2628, 2566] combine_rows_per_src=[5536, 5538, 5536, 5622, 5550, 5494, 5546, 5484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38938 dup_rows=21092 dup_rows_remote=18560 dedup_ratio=0.321838 router_tokens_per_expert=[511, 507, 522, 522, 505, 530, 518, 492, 496, 489, 505, 518, 543, 504, 493, 537, 528, 519, 514, 545, 505, 489, 499, 483, 499, 537, 499, 496, 497, 510, 511, 491, 489, 506, 524, 510, 494, 512, 512, 472, 544, 546, 547, 483, 515, 560, 492, 489, 535, 531, 512, 517, 529, 477, 542, 539, 493, 491, 493, 507, 517, 527, 507, 542] payload_rows_per_dst=[5548, 5528, 5598, 5560, 5506, 5550, 5630, 5524] dup_rows_per_dst=[2666, 2642, 2566, 2520, 2532, 2802, 2734, 2630] combine_rows_per_src=[5548, 5528, 5598, 5560, 5506, 5550, 5630, 5524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=229136 combine_payload_bytes=278462464 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278691600 ingress_bytes=498663424 link_bytes=498663424 token_bytes=557154064 weight_movement_bytes=220200960 total_nvlink_bytes=777355024 weight_to_token_ratio=0.395225 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672577104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278003712 dispatch_meta_bytes=228728 combine_payload_bytes=278003712 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278232440 ingress_bytes=608305152 link_bytes=608305152 token_bytes=556236152 weight_movement_bytes=330301440 total_nvlink_bytes=886537592 weight_to_token_ratio=0.593815 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057942708e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38908 dup_rows=21064 dup_rows_remote=18480 dedup_ratio=0.321411 router_tokens_per_expert=[520, 508, 540, 496, 486, 516, 498, 486, 498, 521, 527, 503, 497, 462, 518, 507, 556, 494, 468, 487, 539, 504, 485, 502, 539, 521, 495, 514, 508, 519, 542, 483, 503, 505, 540, 529, 533, 554, 497, 536, 524, 488, 498, 513, 518, 509, 496, 490, 519, 501, 515, 511, 513, 511, 479, 525, 519, 525, 471, 589, 551, 485, 553, 529] payload_rows_per_dst=[5476, 5494, 5502, 5630, 5700, 5478, 5564, 5628] dup_rows_per_dst=[2624, 2572, 2568, 2612, 2694, 2594, 2584, 2816] combine_rows_per_src=[5476, 5494, 5502, 5630, 5700, 5478, 5564, 5628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277889024 dispatch_meta_bytes=229872 combine_payload_bytes=277889024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278118896 ingress_bytes=387989504 link_bytes=387989504 token_bytes=556007920 weight_movement_bytes=110100480 total_nvlink_bytes=666108400 weight_to_token_ratio=0.198020 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747775608e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38814 dup_rows=21034 dup_rows_remote=18194 dedup_ratio=0.320953 router_tokens_per_expert=[517, 512, 488, 531, 533, 476, 519, 536, 529, 499, 531, 522, 485, 520, 465, 510, 530, 539, 495, 516, 491, 490, 488, 480, 524, 484, 517, 555, 517, 516, 533, 532, 504, 498, 524, 501, 553, 519, 499, 492, 527, 523, 526, 528, 496, 500, 512, 517, 534, 543, 485, 508, 512, 506, 522, 534, 480, 482, 490, 484, 512, 521, 530, 526] payload_rows_per_dst=[5560, 5516, 5488, 5566, 5660, 5418, 5606, 5688] dup_rows_per_dst=[2628, 2606, 2570, 2736, 2610, 2318, 2726, 2840] combine_rows_per_src=[5560, 5516, 5488, 5566, 5660, 5418, 5606, 5688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278218752 dispatch_meta_bytes=228032 combine_payload_bytes=278218752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278446784 ingress_bytes=388319232 link_bytes=388319232 token_bytes=556665536 weight_movement_bytes=110100480 total_nvlink_bytes=666766016 weight_to_token_ratio=0.197786 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753510120e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44566 payload_rows_remote=39006 dup_rows=20970 dup_rows_remote=18412 dedup_ratio=0.319977 router_tokens_per_expert=[505, 483, 480, 507, 502, 509, 533, 519, 459, 517, 525, 520, 481, 522, 514, 497, 523, 515, 510, 509, 512, 517, 516, 521, 555, 493, 520, 501, 490, 497, 478, 525, 533, 480, 511, 513, 546, 504, 572, 511, 525, 487, 544, 531, 526, 482, 492, 543, 486, 515, 523, 525, 513, 506, 544, 532, 544, 490, 501, 542, 483, 514, 501, 494] payload_rows_per_dst=[5592, 5514, 5568, 5560, 5608, 5586, 5680, 5458] dup_rows_per_dst=[2484, 2556, 2678, 2558, 2732, 2674, 2608, 2680] combine_rows_per_src=[5592, 5514, 5568, 5560, 5608, 5586, 5680, 5458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38852 dup_rows=21134 dup_rows_remote=18396 dedup_ratio=0.322479 router_tokens_per_expert=[528, 544, 512, 508, 466, 505, 542, 518, 480, 490, 492, 489, 535, 529, 503, 529, 506, 499, 498, 494, 572, 516, 511, 520, 503, 532, 486, 516, 532, 520, 506, 480, 543, 494, 511, 508, 520, 495, 501, 502, 535, 519, 522, 493, 528, 524, 522, 501, 536, 548, 556, 495, 483, 504, 477, 529, 533, 501, 501, 484, 503, 539, 477, 523] payload_rows_per_dst=[5526, 5494, 5612, 5504, 5586, 5550, 5582, 5548] dup_rows_per_dst=[2720, 2600, 2620, 2646, 2562, 2738, 2674, 2574] combine_rows_per_src=[5526, 5494, 5612, 5504, 5586, 5550, 5582, 5548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38968 dup_rows=20978 dup_rows_remote=18282 dedup_ratio=0.320099 router_tokens_per_expert=[496, 485, 528, 531, 521, 509, 538, 534, 531, 510, 516, 541, 497, 515, 474, 509, 514, 537, 502, 514, 496, 474, 536, 535, 495, 572, 491, 524, 524, 482, 478, 504, 547, 484, 501, 524, 501, 451, 539, 531, 522, 554, 519, 501, 498, 538, 528, 532, 537, 514, 501, 513, 512, 467, 541, 544, 462, 487, 477, 471, 498, 518, 536, 507] payload_rows_per_dst=[5590, 5826, 5710, 5542, 5428, 5664, 5350, 5448] dup_rows_per_dst=[2696, 2836, 2850, 2598, 2384, 2718, 2432, 2464] combine_rows_per_src=[5590, 5826, 5710, 5542, 5428, 5664, 5350, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277889024 dispatch_meta_bytes=229872 combine_payload_bytes=277889024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278118896 ingress_bytes=387989504 link_bytes=387989504 token_bytes=556007920 weight_movement_bytes=110100480 total_nvlink_bytes=666108400 weight_to_token_ratio=0.198020 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747775608e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278892544 dispatch_meta_bytes=229552 combine_payload_bytes=278892544 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279122096 ingress_bytes=499093504 link_bytes=499093504 token_bytes=558014640 weight_movement_bytes=220200960 total_nvlink_bytes=778215600 weight_to_token_ratio=0.394615 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680056902e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229992 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279337576 ingress_bytes=499308544 link_bytes=499308544 token_bytes=558445160 weight_movement_bytes=220200960 total_nvlink_bytes=778646120 weight_to_token_ratio=0.394311 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.683796802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38852 dup_rows=21134 dup_rows_remote=18396 dedup_ratio=0.322479 router_tokens_per_expert=[528, 544, 512, 508, 466, 505, 542, 518, 480, 490, 492, 489, 535, 529, 503, 529, 506, 499, 498, 494, 572, 516, 511, 520, 503, 532, 486, 516, 532, 520, 506, 480, 543, 494, 511, 508, 520, 495, 501, 502, 535, 519, 522, 493, 528, 524, 522, 501, 536, 548, 556, 495, 483, 504, 477, 529, 533, 501, 501, 484, 503, 539, 477, 523] payload_rows_per_dst=[5526, 5494, 5612, 5504, 5586, 5550, 5582, 5548] dup_rows_per_dst=[2720, 2600, 2620, 2646, 2562, 2738, 2674, 2574] combine_rows_per_src=[5526, 5494, 5612, 5504, 5586, 5550, 5582, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=39038 dup_rows=21004 dup_rows_remote=18380 dedup_ratio=0.320496 router_tokens_per_expert=[504, 511, 513, 532, 499, 529, 496, 495, 510, 492, 501, 483, 535, 514, 511, 481, 520, 554, 496, 488, 541, 465, 477, 518, 535, 494, 474, 525, 566, 520, 514, 536, 506, 504, 513, 479, 489, 537, 543, 515, 592, 503, 481, 532, 507, 533, 513, 525, 487, 492, 534, 528, 517, 535, 503, 486, 522, 485, 521, 512, 498, 499, 522, 526] payload_rows_per_dst=[5558, 5462, 5494, 5622, 5596, 5718, 5550, 5532] dup_rows_per_dst=[2600, 2592, 2624, 2706, 2576, 2654, 2614, 2638] combine_rows_per_src=[5558, 5462, 5494, 5622, 5596, 5718, 5550, 5532] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278892544 dispatch_meta_bytes=229552 combine_payload_bytes=278892544 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279122096 ingress_bytes=499093504 link_bytes=499093504 token_bytes=558014640 weight_movement_bytes=220200960 total_nvlink_bytes=778215600 weight_to_token_ratio=0.394615 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680056902e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278218752 dispatch_meta_bytes=228032 combine_payload_bytes=278218752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278446784 ingress_bytes=388319232 link_bytes=388319232 token_bytes=556665536 weight_movement_bytes=110100480 total_nvlink_bytes=666766016 weight_to_token_ratio=0.197786 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.753510120e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=229672 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279824680 ingress_bytes=499795968 link_bytes=499795968 token_bytes=559419688 weight_movement_bytes=220200960 total_nvlink_bytes=779620648 weight_to_token_ratio=0.393624 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.692273907e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229000 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279551624 ingress_bytes=499523584 link_bytes=499523584 token_bytes=558874248 weight_movement_bytes=220200960 total_nvlink_bytes=779075208 weight_to_token_ratio=0.394008 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687536701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=228992 combine_payload_bytes=278491136 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278720128 ingress_bytes=608792576 link_bytes=608792576 token_bytes=557211264 weight_movement_bytes=330301440 total_nvlink_bytes=887512704 weight_to_token_ratio=0.592776 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058790419e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38946 dup_rows=21068 dup_rows_remote=18508 dedup_ratio=0.321472 router_tokens_per_expert=[538, 510, 539, 503, 500, 480, 497, 513, 494, 487, 504, 545, 493, 555, 531, 533, 469, 510, 512, 533, 504, 500, 500, 513, 485, 483, 492, 521, 553, 509, 501, 497, 483, 507, 465, 487, 550, 538, 537, 526, 529, 547, 517, 503, 524, 541, 483, 537, 470, 556, 525, 509, 523, 470, 536, 507, 497, 472, 537, 547, 503, 513, 511, 514] payload_rows_per_dst=[5506, 5590, 5522, 5506, 5554, 5624, 5564, 5602] dup_rows_per_dst=[2654, 2694, 2560, 2576, 2632, 2738, 2628, 2586] combine_rows_per_src=[5506, 5590, 5522, 5506, 5554, 5624, 5564, 5602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38968 dup_rows=20978 dup_rows_remote=18282 dedup_ratio=0.320099 router_tokens_per_expert=[496, 485, 528, 531, 521, 509, 538, 534, 531, 510, 516, 541, 497, 515, 474, 509, 514, 537, 502, 514, 496, 474, 536, 535, 495, 572, 491, 524, 524, 482, 478, 504, 547, 484, 501, 524, 501, 451, 539, 531, 522, 554, 519, 501, 498, 538, 528, 532, 537, 514, 501, 513, 512, 467, 541, 544, 462, 487, 477, 471, 498, 518, 536, 507] payload_rows_per_dst=[5590, 5826, 5710, 5542, 5428, 5664, 5350, 5448] dup_rows_per_dst=[2696, 2836, 2850, 2598, 2384, 2718, 2432, 2464] combine_rows_per_src=[5590, 5826, 5710, 5542, 5428, 5664, 5350, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44240 payload_rows_remote=38628 dup_rows=21296 dup_rows_remote=18516 dedup_ratio=0.324951 router_tokens_per_expert=[509, 477, 487, 526, 482, 484, 528, 507, 496, 514, 545, 479, 492, 528, 509, 540, 489, 524, 481, 556, 523, 552, 507, 483, 479, 494, 506, 473, 524, 529, 537, 523, 471, 521, 520, 486, 519, 535, 558, 564, 517, 505, 501, 503, 519, 507, 515, 550, 512, 501, 537, 492, 514, 522, 485, 505, 511, 546, 532, 535, 496, 523, 470, 513] payload_rows_per_dst=[5488, 5512, 5372, 5524, 5720, 5574, 5438, 5612] dup_rows_per_dst=[2512, 2554, 2450, 2606, 3020, 2660, 2714, 2780] combine_rows_per_src=[5488, 5512, 5372, 5524, 5720, 5574, 5438, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279824384 dispatch_meta_bytes=229672 combine_payload_bytes=279824384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280054056 ingress_bytes=389924864 link_bytes=389924864 token_bytes=559878440 weight_movement_bytes=110100480 total_nvlink_bytes=669978920 weight_to_token_ratio=0.196651 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781434701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44434 payload_rows_remote=38860 dup_rows=21102 dup_rows_remote=18440 dedup_ratio=0.321991 router_tokens_per_expert=[471, 487, 517, 552, 519, 534, 536, 547, 536, 487, 542, 486, 508, 528, 529, 502, 499, 494, 494, 568, 477, 509, 515, 496, 521, 494, 525, 530, 498, 527, 535, 481, 517, 487, 501, 518, 462, 503, 497, 519, 511, 482, 500, 488, 517, 517, 530, 529, 579, 532, 529, 528, 518, 476, 484, 536, 515, 519, 496, 507, 543, 467, 503, 514] payload_rows_per_dst=[5584, 5574, 5514, 5578, 5486, 5606, 5586, 5506] dup_rows_per_dst=[2742, 2662, 2590, 2644, 2522, 2542, 2778, 2622] combine_rows_per_src=[5584, 5574, 5514, 5578, 5486, 5606, 5586, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44634 payload_rows_remote=39092 dup_rows=20902 dup_rows_remote=18294 dedup_ratio=0.318939 router_tokens_per_expert=[511, 492, 514, 551, 502, 507, 471, 515, 505, 538, 499, 513, 481, 516, 508, 526, 519, 497, 503, 564, 531, 493, 522, 561, 513, 505, 511, 535, 494, 517, 504, 529, 495, 526, 565, 509, 500, 523, 498, 498, 528, 485, 494, 478, 491, 529, 531, 518, 519, 515, 512, 483, 488, 523, 515, 520, 505, 532, 509, 475, 498, 500, 547, 512] payload_rows_per_dst=[5544, 5562, 5722, 5604, 5562, 5482, 5542, 5616] dup_rows_per_dst=[2582, 2610, 2658, 2612, 2666, 2626, 2608, 2540] combine_rows_per_src=[5544, 5562, 5722, 5604, 5562, 5482, 5542, 5616] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38956 dup_rows=21086 dup_rows_remote=18476 dedup_ratio=0.321747 router_tokens_per_expert=[482, 511, 497, 485, 508, 540, 508, 493, 519, 521, 507, 537, 518, 504, 500, 543, 517, 510, 510, 531, 509, 505, 514, 558, 522, 480, 497, 526, 481, 477, 494, 545, 473, 475, 525, 508, 504, 508, 523, 536, 516, 512, 489, 522, 531, 540, 547, 518, 478, 532, 468, 512, 535, 499, 544, 516, 526, 503, 495, 499, 505, 533, 529, 518] payload_rows_per_dst=[5512, 5596, 5586, 5528, 5494, 5630, 5622, 5482] dup_rows_per_dst=[2536, 2702, 2722, 2516, 2610, 2720, 2546, 2734] combine_rows_per_src=[5512, 5596, 5586, 5528, 5494, 5630, 5622, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=276885504 dispatch_meta_bytes=228576 combine_payload_bytes=276885504 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277114080 ingress_bytes=497086464 link_bytes=497086464 token_bytes=553999584 weight_movement_bytes=220200960 total_nvlink_bytes=774200544 weight_to_token_ratio=0.397475 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.645151176e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229816 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279394744 ingress_bytes=389265408 link_bytes=389265408 token_bytes=558559672 weight_movement_bytes=110100480 total_nvlink_bytes=668660152 weight_to_token_ratio=0.197115 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769965677e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229000 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279551624 ingress_bytes=499523584 link_bytes=499523584 token_bytes=558874248 weight_movement_bytes=220200960 total_nvlink_bytes=779075208 weight_to_token_ratio=0.394008 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687536701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44634 payload_rows_remote=39092 dup_rows=20902 dup_rows_remote=18294 dedup_ratio=0.318939 router_tokens_per_expert=[511, 492, 514, 551, 502, 507, 471, 515, 505, 538, 499, 513, 481, 516, 508, 526, 519, 497, 503, 564, 531, 493, 522, 561, 513, 505, 511, 535, 494, 517, 504, 529, 495, 526, 565, 509, 500, 523, 498, 498, 528, 485, 494, 478, 491, 529, 531, 518, 519, 515, 512, 483, 488, 523, 515, 520, 505, 532, 509, 475, 498, 500, 547, 512] payload_rows_per_dst=[5544, 5562, 5722, 5604, 5562, 5482, 5542, 5616] dup_rows_per_dst=[2582, 2610, 2658, 2612, 2666, 2626, 2608, 2540] combine_rows_per_src=[5544, 5562, 5722, 5604, 5562, 5482, 5542, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44240 payload_rows_remote=38628 dup_rows=21296 dup_rows_remote=18516 dedup_ratio=0.324951 router_tokens_per_expert=[509, 477, 487, 526, 482, 484, 528, 507, 496, 514, 545, 479, 492, 528, 509, 540, 489, 524, 481, 556, 523, 552, 507, 483, 479, 494, 506, 473, 524, 529, 537, 523, 471, 521, 520, 486, 519, 535, 558, 564, 517, 505, 501, 503, 519, 507, 515, 550, 512, 501, 537, 492, 514, 522, 485, 505, 511, 546, 532, 535, 496, 523, 470, 513] payload_rows_per_dst=[5488, 5512, 5372, 5524, 5720, 5574, 5438, 5612] dup_rows_per_dst=[2512, 2554, 2450, 2606, 3020, 2660, 2714, 2780] combine_rows_per_src=[5488, 5512, 5372, 5524, 5720, 5574, 5438, 5612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278548480 dispatch_meta_bytes=229200 combine_payload_bytes=278548480 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278777680 ingress_bytes=608849920 link_bytes=608849920 token_bytes=557326160 weight_movement_bytes=330301440 total_nvlink_bytes=887627600 weight_to_token_ratio=0.592654 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058890149e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44702 payload_rows_remote=39174 dup_rows=20834 dup_rows_remote=18316 dedup_ratio=0.317902 router_tokens_per_expert=[510, 523, 532, 513, 528, 502, 523, 497, 515, 543, 537, 510, 534, 497, 502, 507, 513, 468, 554, 515, 507, 550, 512, 512, 529, 496, 532, 502, 483, 511, 495, 475, 547, 492, 484, 502, 488, 521, 485, 512, 537, 515, 512, 498, 533, 507, 523, 485, 520, 506, 508, 527, 534, 529, 486, 541, 485, 473, 513, 522, 522, 520, 509, 505] payload_rows_per_dst=[5610, 5582, 5532, 5528, 5504, 5676, 5674, 5596] dup_rows_per_dst=[2646, 2708, 2730, 2518, 2558, 2544, 2628, 2502] combine_rows_per_src=[5610, 5582, 5532, 5528, 5504, 5676, 5674, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278548480 dispatch_meta_bytes=229200 combine_payload_bytes=278548480 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278777680 ingress_bytes=608849920 link_bytes=608849920 token_bytes=557326160 weight_movement_bytes=330301440 total_nvlink_bytes=887627600 weight_to_token_ratio=0.592654 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058890149e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44524 payload_rows_remote=38776 dup_rows=21012 dup_rows_remote=18268 dedup_ratio=0.320618 router_tokens_per_expert=[471, 518, 493, 518, 530, 524, 527, 520, 505, 524, 523, 539, 490, 517, 477, 517, 521, 529, 542, 504, 524, 491, 516, 484, 480, 486, 525, 572, 534, 487, 506, 556, 531, 543, 525, 483, 523, 540, 489, 488, 528, 533, 476, 505, 531, 472, 491, 511, 521, 504, 488, 514, 514, 525, 428, 536, 533, 529, 493, 512, 493, 488, 539, 532] payload_rows_per_dst=[5748, 5474, 5580, 5660, 5632, 5558, 5408, 5464] dup_rows_per_dst=[2744, 2456, 2606, 2912, 2612, 2788, 2372, 2522] combine_rows_per_src=[5748, 5474, 5580, 5660, 5632, 5558, 5408, 5464] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44574 payload_rows_remote=38982 dup_rows=20962 dup_rows_remote=18424 dedup_ratio=0.319855 router_tokens_per_expert=[522, 502, 519, 482, 480, 484, 528, 515, 505, 480, 491, 497, 548, 515, 525, 511, 496, 536, 545, 509, 508, 517, 531, 507, 497, 491, 495, 509, 497, 490, 485, 538, 535, 464, 499, 502, 481, 485, 500, 549, 543, 510, 504, 551, 469, 519, 510, 459, 543, 541, 516, 488, 525, 498, 559, 517, 510, 550, 542, 509, 551, 532, 522, 530] payload_rows_per_dst=[5474, 5540, 5642, 5504, 5506, 5592, 5612, 5704] dup_rows_per_dst=[2590, 2604, 2656, 2500, 2524, 2538, 2762, 2788] combine_rows_per_src=[5474, 5540, 5642, 5504, 5506, 5592, 5612, 5704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280211456 dispatch_meta_bytes=229544 combine_payload_bytes=280211456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280441000 ingress_bytes=500412416 link_bytes=500412416 token_bytes=560652456 weight_movement_bytes=220200960 total_nvlink_bytes=780853416 weight_to_token_ratio=0.392758 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.702994951e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229728 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279466336 ingress_bytes=389337088 link_bytes=389337088 token_bytes=558702944 weight_movement_bytes=110100480 total_nvlink_bytes=668803424 weight_to_token_ratio=0.197064 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771212310e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5569586872 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=2785937080 ingress_bytes=4655357952 link_bytes=4655357952 total_nvlink_bytes=7441295032 weight_to_token_ratio=0.336059 prefetch_share_of_ingress=0.212852 dedup_ratio=0.321643 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.096433153e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38848 dup_rows=21150 dup_rows_remote=18436 dedup_ratio=0.322723 router_tokens_per_expert=[524, 544, 525, 514, 509, 523, 500, 523, 517, 507, 509, 518, 513, 527, 490, 517, 523, 490, 564, 498, 523, 544, 478, 506, 479, 492, 534, 541, 525, 543, 538, 494, 530, 455, 508, 501, 474, 549, 531, 536, 524, 527, 471, 527, 510, 505, 476, 497, 507, 526, 520, 536, 492, 488, 492, 505, 493, 510, 508, 481, 551, 478, 490, 538] payload_rows_per_dst=[5624, 5564, 5538, 5658, 5516, 5456, 5532, 5498] dup_rows_per_dst=[2700, 2632, 2714, 2634, 2652, 2618, 2600, 2600] combine_rows_per_src=[5624, 5564, 5538, 5658, 5516, 5456, 5532, 5498] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280211456 dispatch_meta_bytes=229544 combine_payload_bytes=280211456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280441000 ingress_bytes=500412416 link_bytes=500412416 token_bytes=560652456 weight_movement_bytes=220200960 total_nvlink_bytes=780853416 weight_to_token_ratio=0.392758 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.702994951e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5587400216 prefetch_bytes=1101004800 reduce_bytes=440401920 weight_movement_bytes=1541406720 egress_bytes=2794847768 ingress_bytes=4333959168 link_bytes=4333959168 total_nvlink_bytes=7128806936 weight_to_token_ratio=0.275872 prefetch_share_of_ingress=0.254041 dedup_ratio=0.320889 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=4 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.537467807e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=276885504 dispatch_meta_bytes=228576 combine_payload_bytes=276885504 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277114080 ingress_bytes=497086464 link_bytes=497086464 token_bytes=553999584 weight_movement_bytes=220200960 total_nvlink_bytes=774200544 weight_to_token_ratio=0.397475 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.645151176e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280799232 dispatch_meta_bytes=229960 combine_payload_bytes=280799232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281029192 ingress_bytes=390899712 link_bytes=390899712 token_bytes=561828424 weight_movement_bytes=110100480 total_nvlink_bytes=671928904 weight_to_token_ratio=0.195968 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.798388911e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228176 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278174544 ingress_bytes=388046848 link_bytes=388046848 token_bytes=556120912 weight_movement_bytes=110100480 total_nvlink_bytes=666221392 weight_to_token_ratio=0.197979 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748772914e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229624 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279652600 ingress_bytes=389523456 link_bytes=389523456 token_bytes=559075576 weight_movement_bytes=110100480 total_nvlink_bytes=669176056 weight_to_token_ratio=0.196933 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774453556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584073688 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2793184216 ingress_bytes=4772698112 link_bytes=4772698112 total_nvlink_bytes=7565882328 weight_to_token_ratio=0.354904 prefetch_share_of_ingress=0.207619 dedup_ratio=0.321182 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.300506991e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38868 dup_rows=21064 dup_rows_remote=18492 dedup_ratio=0.321411 router_tokens_per_expert=[507, 514, 509, 476, 555, 532, 530, 489, 528, 550, 517, 522, 553, 513, 515, 502, 493, 509, 548, 491, 501, 495, 525, 526, 512, 486, 509, 524, 514, 538, 534, 534, 498, 486, 520, 519, 510, 522, 513, 492, 496, 497, 529, 483, 507, 489, 483, 516, 501, 469, 530, 501, 535, 546, 536, 505, 515, 510, 509, 527, 483, 508, 482, 500] payload_rows_per_dst=[5528, 5626, 5604, 5606, 5478, 5490, 5598, 5542] dup_rows_per_dst=[2696, 2774, 2572, 2696, 2642, 2510, 2648, 2526] combine_rows_per_src=[5528, 5626, 5604, 5606, 5478, 5490, 5598, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44524 payload_rows_remote=38776 dup_rows=21012 dup_rows_remote=18268 dedup_ratio=0.320618 router_tokens_per_expert=[471, 518, 493, 518, 530, 524, 527, 520, 505, 524, 523, 539, 490, 517, 477, 517, 521, 529, 542, 504, 524, 491, 516, 484, 480, 486, 525, 572, 534, 487, 506, 556, 531, 543, 525, 483, 523, 540, 489, 488, 528, 533, 476, 505, 531, 472, 491, 511, 521, 504, 488, 514, 514, 525, 428, 536, 533, 529, 493, 512, 493, 488, 539, 532] payload_rows_per_dst=[5748, 5474, 5580, 5660, 5632, 5558, 5408, 5464] dup_rows_per_dst=[2744, 2456, 2606, 2912, 2612, 2788, 2372, 2522] combine_rows_per_src=[5748, 5474, 5580, 5660, 5632, 5558, 5408, 5464] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588402296 prefetch_bytes=770703360 reduce_bytes=1321205760 weight_movement_bytes=2091909120 egress_bytes=2795348088 ingress_bytes=4884963328 link_bytes=4885193640 total_nvlink_bytes=7680311416 weight_to_token_ratio=0.374330 prefetch_share_of_ingress=0.157771 dedup_ratio=0.320514 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=12 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.496155216e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5569586872 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=2785937080 ingress_bytes=4655357952 link_bytes=4655357952 total_nvlink_bytes=7441295032 weight_to_token_ratio=0.336059 prefetch_share_of_ingress=0.212852 dedup_ratio=0.321643 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.096433153e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584073688 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2793184216 ingress_bytes=4772698112 link_bytes=4772698112 total_nvlink_bytes=7565882328 weight_to_token_ratio=0.354904 prefetch_share_of_ingress=0.207619 dedup_ratio=0.321182 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.300506991e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=229136 combine_payload_bytes=278462464 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278691600 ingress_bytes=498663424 link_bytes=498663424 token_bytes=557154064 weight_movement_bytes=220200960 total_nvlink_bytes=777355024 weight_to_token_ratio=0.395225 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672577104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5589119472 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=2795706864 ingress_bytes=4444919808 link_bytes=4444919808 total_nvlink_bytes=7240626672 weight_to_token_ratio=0.295486 prefetch_share_of_ingress=0.247700 dedup_ratio=0.320688 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.730446610e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584818536 prefetch_bytes=660602880 reduce_bytes=990904320 weight_movement_bytes=1651507200 egress_bytes=2793556328 ingress_bytes=4442769408 link_bytes=4442769408 total_nvlink_bytes=7236325736 weight_to_token_ratio=0.295714 prefetch_share_of_ingress=0.148692 dedup_ratio=0.321133 prefetch_slots=12 unused_prefetch_slots=148 slot_utilization=0.075000 reduce_slots=9 remote_demand=6 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.726706711e-05 prefetch_bytes_per_gemm_flop=1.148897059e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5570161472 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=2786224960 ingress_bytes=4545544192 link_bytes=4545544192 total_nvlink_bytes=7331769152 weight_to_token_ratio=0.316258 prefetch_share_of_ingress=0.193773 dedup_ratio=0.321582 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.905448963e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38946 dup_rows=21068 dup_rows_remote=18508 dedup_ratio=0.321472 router_tokens_per_expert=[538, 510, 539, 503, 500, 480, 497, 513, 494, 487, 504, 545, 493, 555, 531, 533, 469, 510, 512, 533, 504, 500, 500, 513, 485, 483, 492, 521, 553, 509, 501, 497, 483, 507, 465, 487, 550, 538, 537, 526, 529, 547, 517, 503, 524, 541, 483, 537, 470, 556, 525, 509, 523, 470, 536, 507, 497, 472, 537, 547, 503, 513, 511, 514] payload_rows_per_dst=[5506, 5590, 5522, 5506, 5554, 5624, 5564, 5602] dup_rows_per_dst=[2654, 2694, 2560, 2576, 2632, 2738, 2628, 2586] combine_rows_per_src=[5506, 5590, 5522, 5506, 5554, 5624, 5564, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229816 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279394744 ingress_bytes=389265408 link_bytes=389265408 token_bytes=558559672 weight_movement_bytes=110100480 total_nvlink_bytes=668660152 weight_to_token_ratio=0.197115 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769965677e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38868 dup_rows=21064 dup_rows_remote=18492 dedup_ratio=0.321411 router_tokens_per_expert=[507, 514, 509, 476, 555, 532, 530, 489, 528, 550, 517, 522, 553, 513, 515, 502, 493, 509, 548, 491, 501, 495, 525, 526, 512, 486, 509, 524, 514, 538, 534, 534, 498, 486, 520, 519, 510, 522, 513, 492, 496, 497, 529, 483, 507, 489, 483, 516, 501, 469, 530, 501, 535, 546, 536, 505, 515, 510, 509, 527, 483, 508, 482, 500] payload_rows_per_dst=[5528, 5626, 5604, 5606, 5478, 5490, 5598, 5542] dup_rows_per_dst=[2696, 2774, 2572, 2696, 2642, 2510, 2648, 2526] combine_rows_per_src=[5528, 5626, 5604, 5606, 5478, 5490, 5598, 5542] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=229440 combine_payload_bytes=278605824 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278835264 ingress_bytes=608907264 link_bytes=608907264 token_bytes=557441088 weight_movement_bytes=330301440 total_nvlink_bytes=887742528 weight_to_token_ratio=0.592532 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058989880e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228176 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278174544 ingress_bytes=388046848 link_bytes=388046848 token_bytes=556120912 weight_movement_bytes=110100480 total_nvlink_bytes=666221392 weight_to_token_ratio=0.197979 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748772914e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582669336 prefetch_bytes=880803840 reduce_bytes=1321205760 weight_movement_bytes=2202009600 egress_bytes=2792482328 ingress_bytes=4992196608 link_bytes=4992196608 total_nvlink_bytes=7784678936 weight_to_token_ratio=0.394437 prefetch_share_of_ingress=0.176436 dedup_ratio=0.321530 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=12 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.682250977e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5570161472 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=2786224960 ingress_bytes=4545544192 link_bytes=4545544192 total_nvlink_bytes=7331769152 weight_to_token_ratio=0.316258 prefetch_share_of_ingress=0.193773 dedup_ratio=0.321582 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.905448963e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none - [2026-08-26 16:58:54] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 363.1 | throughput per GPU (TFLOP/s/GPU): 340.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.755975E+00 | loss scale: 1.0 | grad norm: 3.498 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44674 payload_rows_remote=39094 dup_rows=20862 dup_rows_remote=18284 dedup_ratio=0.318329 router_tokens_per_expert=[516, 501, 517, 475, 508, 499, 562, 499, 500, 508, 567, 507, 515, 479, 500, 503, 485, 526, 536, 522, 508, 507, 512, 522, 537, 534, 516, 505, 534, 512, 502, 525, 522, 542, 533, 495, 542, 455, 498, 512, 515, 488, 495, 507, 537, 485, 490, 520, 493, 523, 520, 478, 534, 513, 515, 509, 529, 504, 513, 500, 498, 544, 497, 523] payload_rows_per_dst=[5580, 5580, 5566, 5638, 5592, 5590, 5494, 5634] dup_rows_per_dst=[2574, 2578, 2670, 2692, 2606, 2484, 2676, 2582] combine_rows_per_src=[5580, 5580, 5566, 5638, 5592, 5590, 5494, 5634] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38914 dup_rows=20984 dup_rows_remote=18306 dedup_ratio=0.320190 router_tokens_per_expert=[532, 533, 511, 525, 515, 509, 493, 485, 535, 524, 506, 496, 509, 542, 540, 550, 476, 523, 480, 540, 523, 497, 497, 522, 472, 522, 515, 498, 504, 532, 519, 521, 495, 535, 514, 539, 488, 494, 487, 489, 480, 544, 525, 516, 502, 495, 525, 509, 552, 527, 540, 524, 498, 478, 534, 505, 505, 508, 510, 479, 506, 502, 496, 521] payload_rows_per_dst=[5566, 5626, 5582, 5564, 5442, 5600, 5638, 5534] dup_rows_per_dst=[2640, 2736, 2534, 2602, 2640, 2634, 2678, 2520] combine_rows_per_src=[5566, 5626, 5582, 5564, 5442, 5600, 5638, 5534] + [2026-08-26 18:10:59] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 363.6 | throughput per GPU (TFLOP/s/GPU): 339.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.757419E+00 | loss scale: 1.0 | grad norm: 3.460 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38860 dup_rows=21092 dup_rows_remote=18372 dedup_ratio=0.321838 router_tokens_per_expert=[474, 507, 475, 541, 483, 508, 529, 515, 527, 489, 487, 522, 475, 532, 567, 500, 483, 517, 507, 482, 543, 506, 527, 538, 508, 542, 513, 533, 466, 517, 538, 528, 502, 503, 555, 532, 506, 524, 527, 503, 506, 500, 521, 528, 504, 482, 520, 496, 480, 513, 502, 524, 528, 508, 492, 528, 526, 545, 495, 519, 516, 494, 512, 498] payload_rows_per_dst=[5522, 5528, 5540, 5658, 5584, 5534, 5534, 5544] dup_rows_per_dst=[2542, 2670, 2666, 2632, 2720, 2580, 2616, 2666] combine_rows_per_src=[5522, 5528, 5540, 5658, 5584, 5534, 5534, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44606 payload_rows_remote=39080 dup_rows=20930 dup_rows_remote=18280 dedup_ratio=0.319366 router_tokens_per_expert=[485, 513, 509, 516, 513, 501, 536, 500, 541, 517, 514, 530, 492, 547, 505, 498, 498, 525, 504, 529, 542, 532, 518, 538, 499, 552, 529, 504, 492, 532, 486, 494, 528, 505, 519, 532, 498, 475, 496, 509, 525, 518, 474, 500, 490, 449, 525, 519, 536, 495, 546, 493, 492, 519, 523, 512, 546, 527, 510, 529, 519, 484, 496, 488] payload_rows_per_dst=[5586, 5612, 5646, 5526, 5546, 5534, 5634, 5522] dup_rows_per_dst=[2560, 2676, 2726, 2650, 2578, 2466, 2598, 2676] combine_rows_per_src=[5586, 5612, 5646, 5526, 5546, 5534, 5634, 5522] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38864 dup_rows=21118 dup_rows_remote=18520 dedup_ratio=0.322235 router_tokens_per_expert=[507, 515, 511, 543, 554, 494, 517, 468, 547, 526, 504, 500, 543, 508, 521, 525, 511, 529, 526, 517, 553, 530, 504, 479, 553, 506, 478, 496, 534, 479, 523, 492, 478, 458, 508, 504, 521, 535, 497, 516, 475, 530, 509, 483, 491, 493, 522, 535, 554, 488, 522, 524, 505, 486, 530, 550, 495, 500, 460, 502, 526, 544, 507, 527] payload_rows_per_dst=[5574, 5150, 5648, 5346, 5674, 5938, 5534, 5554] dup_rows_per_dst=[2644, 2156, 2650, 2386, 2750, 3180, 2754, 2598] combine_rows_per_src=[5574, 5150, 5648, 5346, 5674, 5938, 5534, 5554] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38922 dup_rows=21082 dup_rows_remote=18596 dedup_ratio=0.321686 router_tokens_per_expert=[507, 525, 529, 521, 518, 495, 526, 513, 485, 536, 542, 509, 545, 484, 478, 536, 530, 495, 553, 478, 515, 511, 477, 500, 526, 491, 555, 512, 523, 502, 560, 521, 520, 463, 505, 480, 539, 519, 534, 486, 498, 514, 522, 508, 471, 524, 476, 496, 514, 495, 513, 489, 516, 538, 490, 504, 507, 515, 493, 501, 585, 537, 528, 490] payload_rows_per_dst=[5596, 5482, 5540, 5710, 5452, 5532, 5522, 5620] dup_rows_per_dst=[2672, 2748, 2578, 2670, 2640, 2486, 2596, 2692] combine_rows_per_src=[5596, 5482, 5540, 5710, 5452, 5532, 5522, 5620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44606 payload_rows_remote=39080 dup_rows=20930 dup_rows_remote=18280 dedup_ratio=0.319366 router_tokens_per_expert=[485, 513, 509, 516, 513, 501, 536, 500, 541, 517, 514, 530, 492, 547, 505, 498, 498, 525, 504, 529, 542, 532, 518, 538, 499, 552, 529, 504, 492, 532, 486, 494, 528, 505, 519, 532, 498, 475, 496, 509, 525, 518, 474, 500, 490, 449, 525, 519, 536, 495, 546, 493, 492, 519, 523, 512, 546, 527, 510, 529, 519, 484, 496, 488] payload_rows_per_dst=[5586, 5612, 5646, 5526, 5546, 5534, 5634, 5522] dup_rows_per_dst=[2560, 2676, 2726, 2650, 2578, 2466, 2598, 2676] combine_rows_per_src=[5586, 5612, 5646, 5526, 5546, 5534, 5634, 5522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38914 dup_rows=20984 dup_rows_remote=18306 dedup_ratio=0.320190 router_tokens_per_expert=[532, 533, 511, 525, 515, 509, 493, 485, 535, 524, 506, 496, 509, 542, 540, 550, 476, 523, 480, 540, 523, 497, 497, 522, 472, 522, 515, 498, 504, 532, 519, 521, 495, 535, 514, 539, 488, 494, 487, 489, 480, 544, 525, 516, 502, 495, 525, 509, 552, 527, 540, 524, 498, 478, 534, 505, 505, 508, 510, 479, 506, 502, 496, 521] payload_rows_per_dst=[5566, 5626, 5582, 5564, 5442, 5600, 5638, 5534] dup_rows_per_dst=[2640, 2736, 2534, 2602, 2640, 2634, 2678, 2520] combine_rows_per_src=[5566, 5626, 5582, 5564, 5442, 5600, 5638, 5534] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38894 dup_rows=20976 dup_rows_remote=18304 dedup_ratio=0.320068 router_tokens_per_expert=[568, 518, 538, 521, 518, 503, 483, 476, 532, 474, 501, 510, 518, 524, 545, 521, 518, 545, 510, 493, 521, 541, 544, 497, 497, 505, 531, 541, 521, 526, 487, 490, 496, 513, 467, 528, 530, 490, 523, 548, 480, 491, 514, 508, 499, 549, 493, 513, 503, 517, 493, 475, 521, 512, 511, 509, 556, 514, 494, 489, 506, 500, 488, 521] payload_rows_per_dst=[5606, 5592, 5666, 5586, 5606, 5494, 5492, 5518] dup_rows_per_dst=[2644, 2658, 2672, 2610, 2584, 2600, 2590, 2618] combine_rows_per_src=[5606, 5592, 5666, 5586, 5606, 5494, 5492, 5518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=229512 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280455304 ingress_bytes=500426752 link_bytes=500426752 token_bytes=560681096 weight_movement_bytes=220200960 total_nvlink_bytes=780882056 weight_to_token_ratio=0.392738 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.703244278e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=39020 dup_rows=20990 dup_rows_remote=18286 dedup_ratio=0.320282 router_tokens_per_expert=[550, 482, 508, 508, 528, 484, 500, 523, 490, 540, 501, 531, 526, 511, 521, 520, 546, 527, 485, 505, 531, 503, 504, 479, 479, 513, 484, 528, 526, 485, 525, 543, 543, 503, 483, 512, 540, 513, 465, 502, 528, 475, 536, 460, 473, 485, 493, 492, 526, 510, 544, 520, 512, 531, 482, 543, 526, 526, 541, 533, 548, 516, 497, 524] payload_rows_per_dst=[5526, 5776, 5438, 5596, 5518, 5494, 5570, 5628] dup_rows_per_dst=[2704, 2968, 2414, 2670, 2604, 2390, 2610, 2630] combine_rows_per_src=[5526, 5776, 5438, 5596, 5518, 5494, 5570, 5628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=228880 combine_payload_bytes=278935552 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279164432 ingress_bytes=499136512 link_bytes=499136512 token_bytes=558099984 weight_movement_bytes=220200960 total_nvlink_bytes=778300944 weight_to_token_ratio=0.394555 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680804882e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229536 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278806688 ingress_bytes=608878592 link_bytes=608878592 token_bytes=557383840 weight_movement_bytes=330301440 total_nvlink_bytes=887685280 weight_to_token_ratio=0.592592 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058940015e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278548480 dispatch_meta_bytes=228928 combine_payload_bytes=278548480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278777408 ingress_bytes=388648960 link_bytes=388648960 token_bytes=557325888 weight_movement_bytes=110100480 total_nvlink_bytes=667426368 weight_to_token_ratio=0.197551 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759244632e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=230072 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279222968 ingress_bytes=389093376 link_bytes=389093376 token_bytes=558215864 weight_movement_bytes=110100480 total_nvlink_bytes=668316344 weight_to_token_ratio=0.197236 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766973757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280125440 dispatch_meta_bytes=229440 combine_payload_bytes=280125440 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280354880 ingress_bytes=500326400 link_bytes=500326400 token_bytes=560480320 weight_movement_bytes=220200960 total_nvlink_bytes=780681280 weight_to_token_ratio=0.392879 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701498992e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229536 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278806688 ingress_bytes=608878592 link_bytes=608878592 token_bytes=557383840 weight_movement_bytes=330301440 total_nvlink_bytes=887685280 weight_to_token_ratio=0.592592 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058940015e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=230072 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279222968 ingress_bytes=389093376 link_bytes=389093376 token_bytes=558215864 weight_movement_bytes=110100480 total_nvlink_bytes=668316344 weight_to_token_ratio=0.197236 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766973757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280125440 dispatch_meta_bytes=229440 combine_payload_bytes=280125440 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280354880 ingress_bytes=500326400 link_bytes=500326400 token_bytes=560480320 weight_movement_bytes=220200960 total_nvlink_bytes=780681280 weight_to_token_ratio=0.392879 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701498992e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=228880 combine_payload_bytes=278935552 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279164432 ingress_bytes=499136512 link_bytes=499136512 token_bytes=558099984 weight_movement_bytes=220200960 total_nvlink_bytes=778300944 weight_to_token_ratio=0.394555 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680804882e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=228792 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279020984 ingress_bytes=498993152 link_bytes=498993152 token_bytes=557813176 weight_movement_bytes=220200960 total_nvlink_bytes=778014136 weight_to_token_ratio=0.394758 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678311616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279695360 dispatch_meta_bytes=229224 combine_payload_bytes=279695360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279924584 ingress_bytes=389795840 link_bytes=389795840 token_bytes=559619944 weight_movement_bytes=110100480 total_nvlink_bytes=669720424 weight_to_token_ratio=0.196742 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779190762e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=39008 dup_rows=20950 dup_rows_remote=18364 dedup_ratio=0.319672 router_tokens_per_expert=[489, 503, 500, 519, 535, 525, 508, 530, 523, 497, 510, 513, 507, 497, 522, 513, 509, 529, 508, 564, 535, 510, 504, 498, 497, 507, 519, 525, 497, 532, 508, 511, 468, 504, 539, 501, 499, 470, 497, 519, 492, 500, 536, 508, 481, 497, 527, 500, 516, 493, 488, 506, 508, 528, 494, 522, 533, 493, 551, 530, 515, 541, 538, 530] payload_rows_per_dst=[5578, 5578, 5602, 5608, 5452, 5534, 5568, 5666] dup_rows_per_dst=[2640, 2586, 2712, 2584, 2542, 2548, 2542, 2796] combine_rows_per_src=[5578, 5578, 5602, 5608, 5452, 5534, 5568, 5666] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44358 payload_rows_remote=38724 dup_rows=21178 dup_rows_remote=18466 dedup_ratio=0.323151 router_tokens_per_expert=[531, 466, 506, 522, 515, 486, 561, 505, 525, 548, 534, 537, 516, 539, 456, 538, 525, 473, 484, 524, 470, 534, 486, 517, 500, 511, 479, 500, 476, 547, 541, 504, 549, 470, 495, 515, 504, 518, 482, 497, 542, 544, 474, 480, 514, 528, 502, 555, 495, 524, 522, 517, 520, 523, 514, 558, 513, 515, 506, 523, 514, 511, 466, 522] payload_rows_per_dst=[5596, 5678, 5446, 5496, 5434, 5530, 5634, 5544] dup_rows_per_dst=[2588, 2708, 2580, 2620, 2626, 2748, 2712, 2596] combine_rows_per_src=[5596, 5678, 5446, 5496, 5434, 5530, 5634, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44280 payload_rows_remote=38844 dup_rows=21256 dup_rows_remote=18720 dedup_ratio=0.324341 router_tokens_per_expert=[513, 535, 540, 498, 503, 504, 499, 538, 535, 503, 512, 509, 476, 520, 520, 517, 500, 528, 494, 499, 555, 491, 497, 522, 496, 486, 507, 482, 480, 514, 542, 479, 526, 518, 488, 517, 472, 539, 513, 529, 479, 499, 535, 490, 486, 530, 494, 475, 518, 513, 530, 533, 535, 512, 525, 495, 517, 565, 518, 537, 491, 539, 508, 548] payload_rows_per_dst=[5574, 5488, 5530, 5436, 5496, 5506, 5622, 5628] dup_rows_per_dst=[2686, 2696, 2642, 2536, 2708, 2470, 2700, 2818] combine_rows_per_src=[5574, 5488, 5530, 5436, 5496, 5506, 5622, 5628] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=39028 dup_rows=20918 dup_rows_remote=18306 dedup_ratio=0.319183 router_tokens_per_expert=[539, 511, 532, 498, 500, 498, 536, 510, 503, 488, 504, 543, 510, 506, 514, 511, 510, 504, 538, 546, 509, 543, 533, 495, 489, 516, 499, 504, 520, 525, 527, 517, 520, 528, 539, 483, 470, 546, 494, 521, 489, 482, 490, 506, 500, 516, 541, 509, 524, 513, 518, 524, 472, 535, 482, 495, 568, 487, 526, 522, 480, 490, 491, 529] payload_rows_per_dst=[5592, 5518, 5664, 5596, 5590, 5618, 5520, 5520] dup_rows_per_dst=[2656, 2640, 2692, 2598, 2612, 2448, 2606, 2666] combine_rows_per_src=[5592, 5518, 5664, 5596, 5590, 5618, 5520, 5520] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38852 dup_rows=21086 dup_rows_remote=18350 dedup_ratio=0.321747 router_tokens_per_expert=[526, 514, 528, 506, 503, 485, 532, 494, 507, 495, 501, 507, 490, 539, 512, 499, 539, 546, 512, 501, 491, 516, 536, 502, 506, 525, 498, 535, 512, 470, 505, 502, 526, 521, 527, 505, 491, 522, 512, 520, 514, 495, 498, 498, 526, 471, 508, 478, 500, 560, 541, 542, 509, 493, 470, 543, 504, 500, 491, 530, 551, 542, 502, 544] payload_rows_per_dst=[5406, 5534, 5652, 5456, 5638, 5564, 5602, 5598] dup_rows_per_dst=[2472, 2566, 2634, 2650, 2610, 2710, 2708, 2736] combine_rows_per_src=[5406, 5534, 5652, 5456, 5638, 5564, 5602, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44674 payload_rows_remote=39094 dup_rows=20862 dup_rows_remote=18284 dedup_ratio=0.318329 router_tokens_per_expert=[516, 501, 517, 475, 508, 499, 562, 499, 500, 508, 567, 507, 515, 479, 500, 503, 485, 526, 536, 522, 508, 507, 512, 522, 537, 534, 516, 505, 534, 512, 502, 525, 522, 542, 533, 495, 542, 455, 498, 512, 515, 488, 495, 507, 537, 485, 490, 520, 493, 523, 520, 478, 534, 513, 515, 509, 529, 504, 513, 500, 498, 544, 497, 523] payload_rows_per_dst=[5580, 5580, 5566, 5638, 5592, 5590, 5494, 5634] dup_rows_per_dst=[2574, 2578, 2670, 2692, 2606, 2484, 2676, 2582] combine_rows_per_src=[5580, 5580, 5566, 5638, 5592, 5590, 5494, 5634] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38828 dup_rows=21114 dup_rows_remote=18400 dedup_ratio=0.322174 router_tokens_per_expert=[503, 529, 496, 539, 496, 516, 552, 522, 479, 484, 504, 530, 506, 491, 499, 484, 513, 542, 497, 516, 543, 480, 541, 513, 501, 521, 547, 528, 489, 495, 498, 502, 517, 496, 506, 504, 497, 530, 491, 531, 551, 542, 499, 533, 512, 537, 499, 481, 506, 495, 459, 454, 523, 508, 498, 498, 529, 511, 499, 545, 555, 529, 532, 545] payload_rows_per_dst=[5596, 5482, 5622, 5522, 5550, 5594, 5388, 5668] dup_rows_per_dst=[2710, 2472, 2668, 2640, 2594, 2714, 2494, 2822] combine_rows_per_src=[5596, 5482, 5622, 5522, 5550, 5594, 5388, 5668] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279609344 dispatch_meta_bytes=229488 combine_payload_bytes=279609344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279838832 ingress_bytes=389709824 link_bytes=389709824 token_bytes=559448176 weight_movement_bytes=110100480 total_nvlink_bytes=669548656 weight_to_token_ratio=0.196802 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777694802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=228760 combine_payload_bytes=277573632 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277802392 ingress_bytes=497774592 link_bytes=497774592 token_bytes=555376024 weight_movement_bytes=220200960 total_nvlink_bytes=775576984 weight_to_token_ratio=0.396490 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.657118853e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44280 payload_rows_remote=38844 dup_rows=21256 dup_rows_remote=18720 dedup_ratio=0.324341 router_tokens_per_expert=[513, 535, 540, 498, 503, 504, 499, 538, 535, 503, 512, 509, 476, 520, 520, 517, 500, 528, 494, 499, 555, 491, 497, 522, 496, 486, 507, 482, 480, 514, 542, 479, 526, 518, 488, 517, 472, 539, 513, 529, 479, 499, 535, 490, 486, 530, 494, 475, 518, 513, 530, 533, 535, 512, 525, 495, 517, 565, 518, 537, 491, 539, 508, 548] payload_rows_per_dst=[5574, 5488, 5530, 5436, 5496, 5506, 5622, 5628] dup_rows_per_dst=[2686, 2696, 2642, 2536, 2708, 2470, 2700, 2818] combine_rows_per_src=[5574, 5488, 5530, 5436, 5496, 5506, 5622, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44358 payload_rows_remote=38724 dup_rows=21178 dup_rows_remote=18466 dedup_ratio=0.323151 router_tokens_per_expert=[531, 466, 506, 522, 515, 486, 561, 505, 525, 548, 534, 537, 516, 539, 456, 538, 525, 473, 484, 524, 470, 534, 486, 517, 500, 511, 479, 500, 476, 547, 541, 504, 549, 470, 495, 515, 504, 518, 482, 497, 542, 544, 474, 480, 514, 528, 502, 555, 495, 524, 522, 517, 520, 523, 514, 558, 513, 515, 506, 523, 514, 511, 466, 522] payload_rows_per_dst=[5596, 5678, 5446, 5496, 5434, 5530, 5634, 5544] dup_rows_per_dst=[2588, 2708, 2580, 2620, 2626, 2748, 2712, 2596] combine_rows_per_src=[5596, 5678, 5446, 5496, 5434, 5530, 5634, 5544] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=230256 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278664048 ingress_bytes=388534272 link_bytes=388534272 token_bytes=557097840 weight_movement_bytes=110100480 total_nvlink_bytes=667198320 weight_to_token_ratio=0.197632 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757250019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=39018 dup_rows=20954 dup_rows_remote=18296 dedup_ratio=0.319733 router_tokens_per_expert=[510, 494, 521, 504, 471, 527, 516, 496, 484, 496, 506, 512, 493, 531, 503, 537, 526, 479, 529, 516, 511, 486, 481, 526, 522, 506, 529, 499, 500, 524, 527, 530, 515, 494, 510, 510, 513, 486, 543, 525, 536, 472, 512, 518, 515, 512, 521, 535, 541, 477, 531, 518, 521, 513, 479, 519, 503, 529, 541, 496, 531, 526, 509, 525] payload_rows_per_dst=[5564, 5556, 5486, 5740, 5710, 5602, 5622, 5302] dup_rows_per_dst=[2658, 2654, 2492, 2948, 2802, 2640, 2562, 2198] combine_rows_per_src=[5564, 5556, 5486, 5740, 5710, 5602, 5622, 5302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39082 dup_rows=20968 dup_rows_remote=18340 dedup_ratio=0.319946 router_tokens_per_expert=[567, 501, 476, 523, 503, 517, 497, 526, 516, 515, 524, 532, 494, 563, 548, 498, 482, 480, 503, 540, 512, 514, 518, 508, 515, 499, 493, 505, 531, 498, 522, 490, 492, 513, 481, 520, 503, 526, 533, 518, 476, 517, 508, 493, 558, 531, 533, 539, 513, 518, 522, 506, 526, 503, 531, 509, 508, 506, 461, 526, 466, 514, 506, 502] payload_rows_per_dst=[5616, 5626, 5486, 5570, 5598, 5582, 5608, 5482] dup_rows_per_dst=[2604, 2754, 2628, 2536, 2574, 2728, 2648, 2496] combine_rows_per_src=[5616, 5626, 5486, 5570, 5598, 5582, 5608, 5482] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=229336 combine_payload_bytes=279752704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279982040 ingress_bytes=389853184 link_bytes=389853184 token_bytes=559734744 weight_movement_bytes=110100480 total_nvlink_bytes=669835224 weight_to_token_ratio=0.196701 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780188068e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=228808 combine_payload_bytes=278491136 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278719944 ingress_bytes=718893056 link_bytes=718893056 token_bytes=557211080 weight_movement_bytes=440401920 total_nvlink_bytes=997613000 weight_to_token_ratio=0.790368 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250273262e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39082 dup_rows=20968 dup_rows_remote=18340 dedup_ratio=0.319946 router_tokens_per_expert=[567, 501, 476, 523, 503, 517, 497, 526, 516, 515, 524, 532, 494, 563, 548, 498, 482, 480, 503, 540, 512, 514, 518, 508, 515, 499, 493, 505, 531, 498, 522, 490, 492, 513, 481, 520, 503, 526, 533, 518, 476, 517, 508, 493, 558, 531, 533, 539, 513, 518, 522, 506, 526, 503, 531, 509, 508, 506, 461, 526, 466, 514, 506, 502] payload_rows_per_dst=[5616, 5626, 5486, 5570, 5598, 5582, 5608, 5482] dup_rows_per_dst=[2604, 2754, 2628, 2536, 2574, 2728, 2648, 2496] combine_rows_per_src=[5616, 5626, 5486, 5570, 5598, 5582, 5608, 5482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278319104 dispatch_meta_bytes=228912 combine_payload_bytes=278319104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278548016 ingress_bytes=388419584 link_bytes=388419584 token_bytes=556867120 weight_movement_bytes=110100480 total_nvlink_bytes=666967600 weight_to_token_ratio=0.197714 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755255406e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=228760 combine_payload_bytes=277573632 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277802392 ingress_bytes=497774592 link_bytes=497774592 token_bytes=555376024 weight_movement_bytes=220200960 total_nvlink_bytes=775576984 weight_to_token_ratio=0.396490 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.657118853e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44184 payload_rows_remote=38594 dup_rows=21352 dup_rows_remote=18702 dedup_ratio=0.325806 router_tokens_per_expert=[474, 550, 546, 547, 501, 516, 521, 485, 499, 537, 502, 505, 508, 518, 522, 529, 499, 556, 488, 512, 539, 515, 488, 511, 495, 517, 479, 547, 507, 522, 521, 512, 504, 503, 499, 529, 495, 514, 485, 476, 539, 549, 492, 551, 524, 471, 523, 499, 493, 525, 506, 532, 487, 507, 537, 526, 503, 483, 502, 534, 484, 492, 502, 534] payload_rows_per_dst=[5576, 5590, 5566, 5496, 5468, 5518, 5524, 5446] dup_rows_per_dst=[2704, 2650, 2650, 2704, 2542, 2778, 2702, 2622] combine_rows_per_src=[5576, 5590, 5566, 5496, 5468, 5518, 5524, 5446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278433792 dispatch_meta_bytes=230256 combine_payload_bytes=278433792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278664048 ingress_bytes=388534272 link_bytes=388534272 token_bytes=557097840 weight_movement_bytes=110100480 total_nvlink_bytes=667198320 weight_to_token_ratio=0.197632 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.757250019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229256 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279910280 ingress_bytes=499881984 link_bytes=499881984 token_bytes=559591304 weight_movement_bytes=220200960 total_nvlink_bytes=779792264 weight_to_token_ratio=0.393503 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693769866e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38890 dup_rows=20926 dup_rows_remote=18196 dedup_ratio=0.319305 router_tokens_per_expert=[462, 511, 539, 523, 461, 524, 516, 522, 518, 543, 536, 488, 500, 522, 508, 514, 486, 470, 545, 492, 489, 526, 525, 512, 488, 512, 464, 489, 490, 504, 502, 511, 512, 493, 498, 539, 497, 528, 534, 519, 493, 504, 545, 527, 509, 480, 549, 482, 544, 549, 527, 517, 525, 531, 532, 500, 516, 544, 528, 526, 521, 524, 478, 505] payload_rows_per_dst=[5508, 5650, 5544, 5486, 5606, 5516, 5720, 5580] dup_rows_per_dst=[2608, 2608, 2546, 2434, 2634, 2662, 2730, 2704] combine_rows_per_src=[5508, 5650, 5544, 5486, 5606, 5516, 5720, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=229512 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280455304 ingress_bytes=500426752 link_bytes=500426752 token_bytes=560681096 weight_movement_bytes=220200960 total_nvlink_bytes=780882056 weight_to_token_ratio=0.392738 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.703244278e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280139776 dispatch_meta_bytes=229688 combine_payload_bytes=280139776 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280369464 ingress_bytes=500340736 link_bytes=500340736 token_bytes=560509240 weight_movement_bytes=220200960 total_nvlink_bytes=780710200 weight_to_token_ratio=0.392859 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701748318e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38900 dup_rows=21160 dup_rows_remote=18540 dedup_ratio=0.322876 router_tokens_per_expert=[514, 524, 526, 526, 508, 496, 505, 499, 500, 524, 527, 492, 494, 500, 517, 523, 498, 515, 508, 473, 507, 511, 534, 530, 542, 541, 545, 522, 476, 507, 531, 531, 479, 521, 514, 508, 493, 525, 494, 514, 520, 500, 522, 503, 526, 570, 518, 490, 509, 514, 487, 510, 467, 506, 482, 529, 515, 536, 488, 502, 540, 528, 510, 502] payload_rows_per_dst=[5556, 5510, 5540, 5686, 5476, 5592, 5462, 5554] dup_rows_per_dst=[2640, 2644, 2612, 2704, 2620, 2706, 2546, 2688] combine_rows_per_src=[5556, 5510, 5540, 5686, 5476, 5592, 5462, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276641792 dispatch_meta_bytes=229184 combine_payload_bytes=276641792 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=276870976 ingress_bytes=606943232 link_bytes=606943232 token_bytes=553512768 weight_movement_bytes=330301440 total_nvlink_bytes=883814208 weight_to_token_ratio=0.596737 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055574105e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38836 dup_rows=21090 dup_rows_remote=18380 dedup_ratio=0.321808 router_tokens_per_expert=[520, 530, 494, 512, 471, 552, 523, 493, 537, 524, 499, 512, 504, 551, 491, 534, 489, 537, 463, 504, 500, 477, 507, 518, 548, 461, 525, 534, 495, 492, 577, 528, 487, 489, 520, 496, 508, 536, 512, 510, 484, 474, 537, 458, 497, 526, 514, 516, 486, 531, 519, 524, 512, 539, 517, 540, 534, 493, 511, 529, 534, 531, 506, 496] payload_rows_per_dst=[5558, 5654, 5474, 5610, 5498, 5448, 5638, 5566] dup_rows_per_dst=[2632, 2650, 2516, 2710, 2618, 2564, 2698, 2702] combine_rows_per_src=[5558, 5654, 5474, 5610, 5498, 5448, 5638, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38860 dup_rows=21046 dup_rows_remote=18330 dedup_ratio=0.321136 router_tokens_per_expert=[522, 507, 500, 504, 525, 538, 504, 508, 492, 549, 483, 450, 494, 530, 514, 519, 518, 505, 544, 495, 509, 458, 500, 500, 517, 505, 507, 528, 551, 528, 514, 557, 545, 500, 507, 496, 512, 479, 490, 507, 514, 535, 505, 520, 536, 520, 514, 504, 511, 520, 452, 508, 535, 503, 518, 556, 522, 502, 500, 515, 500, 535, 507, 525] payload_rows_per_dst=[5570, 5472, 5492, 5678, 5540, 5570, 5538, 5630] dup_rows_per_dst=[2646, 2526, 2566, 2736, 2532, 2740, 2584, 2716] combine_rows_per_src=[5570, 5472, 5492, 5678, 5540, 5570, 5538, 5630] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=228344 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278991864 ingress_bytes=609064960 link_bytes=609064960 token_bytes=557755384 weight_movement_bytes=330301440 total_nvlink_bytes=888056824 weight_to_token_ratio=0.592198 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059264139e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38900 dup_rows=21160 dup_rows_remote=18540 dedup_ratio=0.322876 router_tokens_per_expert=[514, 524, 526, 526, 508, 496, 505, 499, 500, 524, 527, 492, 494, 500, 517, 523, 498, 515, 508, 473, 507, 511, 534, 530, 542, 541, 545, 522, 476, 507, 531, 531, 479, 521, 514, 508, 493, 525, 494, 514, 520, 500, 522, 503, 526, 570, 518, 490, 509, 514, 487, 510, 467, 506, 482, 529, 515, 536, 488, 502, 540, 528, 510, 502] payload_rows_per_dst=[5556, 5510, 5540, 5686, 5476, 5592, 5462, 5554] dup_rows_per_dst=[2640, 2644, 2612, 2704, 2620, 2706, 2546, 2688] combine_rows_per_src=[5556, 5510, 5540, 5686, 5476, 5592, 5462, 5554] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44744 payload_rows_remote=39094 dup_rows=20792 dup_rows_remote=18132 dedup_ratio=0.317261 router_tokens_per_expert=[527, 475, 481, 525, 519, 501, 515, 533, 516, 555, 490, 508, 511, 527, 506, 499, 543, 503, 522, 488, 513, 500, 514, 513, 502, 517, 470, 509, 490, 504, 506, 524, 527, 505, 474, 543, 495, 503, 504, 571, 509, 542, 508, 545, 518, 482, 522, 529, 492, 533, 510, 510, 501, 511, 545, 504, 531, 512, 522, 464, 517, 531, 510, 492] payload_rows_per_dst=[5514, 5606, 5648, 5558, 5618, 5650, 5556, 5594] dup_rows_per_dst=[2638, 2618, 2544, 2486, 2626, 2660, 2656, 2564] combine_rows_per_src=[5514, 5606, 5648, 5558, 5618, 5650, 5556, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278376448 dispatch_meta_bytes=228864 combine_payload_bytes=278376448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278605312 ingress_bytes=388476928 link_bytes=388476928 token_bytes=556981760 weight_movement_bytes=110100480 total_nvlink_bytes=667082240 weight_to_token_ratio=0.197673 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756252713e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44610 payload_rows_remote=38890 dup_rows=20926 dup_rows_remote=18196 dedup_ratio=0.319305 router_tokens_per_expert=[462, 511, 539, 523, 461, 524, 516, 522, 518, 543, 536, 488, 500, 522, 508, 514, 486, 470, 545, 492, 489, 526, 525, 512, 488, 512, 464, 489, 490, 504, 502, 511, 512, 493, 498, 539, 497, 528, 534, 519, 493, 504, 545, 527, 509, 480, 549, 482, 544, 549, 527, 517, 525, 531, 532, 500, 516, 544, 528, 526, 521, 524, 478, 505] payload_rows_per_dst=[5508, 5650, 5544, 5486, 5606, 5516, 5720, 5580] dup_rows_per_dst=[2608, 2608, 2546, 2434, 2634, 2662, 2730, 2704] combine_rows_per_src=[5508, 5650, 5544, 5486, 5606, 5516, 5720, 5580] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=229760 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279064960 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557900160 weight_movement_bytes=110100480 total_nvlink_bytes=668000640 weight_to_token_ratio=0.197348 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278548480 dispatch_meta_bytes=228760 combine_payload_bytes=278548480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278777240 ingress_bytes=388648960 link_bytes=388648960 token_bytes=557325720 weight_movement_bytes=110100480 total_nvlink_bytes=667426200 weight_to_token_ratio=0.197551 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759244632e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38836 dup_rows=21090 dup_rows_remote=18380 dedup_ratio=0.321808 router_tokens_per_expert=[520, 530, 494, 512, 471, 552, 523, 493, 537, 524, 499, 512, 504, 551, 491, 534, 489, 537, 463, 504, 500, 477, 507, 518, 548, 461, 525, 534, 495, 492, 577, 528, 487, 489, 520, 496, 508, 536, 512, 510, 484, 474, 537, 458, 497, 526, 514, 516, 486, 531, 519, 524, 512, 539, 517, 540, 534, 493, 511, 529, 534, 531, 506, 496] payload_rows_per_dst=[5558, 5654, 5474, 5610, 5498, 5448, 5638, 5566] dup_rows_per_dst=[2632, 2650, 2516, 2710, 2618, 2564, 2698, 2702] combine_rows_per_src=[5558, 5654, 5474, 5610, 5498, 5448, 5638, 5566] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38978 dup_rows=20976 dup_rows_remote=18320 dedup_ratio=0.320068 router_tokens_per_expert=[517, 515, 502, 518, 511, 482, 483, 508, 553, 513, 536, 518, 543, 510, 502, 526, 549, 532, 524, 555, 493, 471, 529, 503, 512, 499, 508, 531, 511, 494, 482, 501, 480, 543, 527, 480, 544, 543, 496, 485, 483, 510, 538, 514, 495, 513, 551, 498, 510, 513, 531, 513, 529, 496, 541, 473, 506, 515, 477, 524, 539, 495, 490, 485] payload_rows_per_dst=[5582, 5582, 5740, 5576, 5724, 5400, 5426, 5530] dup_rows_per_dst=[2656, 2654, 2878, 2532, 2884, 2360, 2480, 2532] combine_rows_per_src=[5582, 5582, 5740, 5576, 5724, 5400, 5426, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278548480 dispatch_meta_bytes=228760 combine_payload_bytes=278548480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278777240 ingress_bytes=388648960 link_bytes=388648960 token_bytes=557325720 weight_movement_bytes=110100480 total_nvlink_bytes=667426200 weight_to_token_ratio=0.197551 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759244632e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=229760 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279064960 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557900160 weight_movement_bytes=110100480 total_nvlink_bytes=668000640 weight_to_token_ratio=0.197348 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44330 payload_rows_remote=38738 dup_rows=21206 dup_rows_remote=18578 dedup_ratio=0.323578 router_tokens_per_expert=[509, 534, 482, 504, 515, 474, 514, 542, 506, 525, 543, 490, 503, 502, 491, 501, 536, 527, 491, 492, 507, 539, 517, 501, 496, 503, 500, 532, 494, 518, 492, 535, 521, 497, 524, 539, 517, 515, 470, 492, 550, 532, 549, 552, 490, 515, 486, 512, 516, 499, 550, 521, 489, 548, 492, 523, 501, 500, 511, 453, 527, 532, 519, 511] payload_rows_per_dst=[5584, 5538, 5592, 5470, 5524, 5614, 5528, 5480] dup_rows_per_dst=[2564, 2584, 2628, 2670, 2626, 2758, 2748, 2628] combine_rows_per_src=[5584, 5538, 5592, 5470, 5524, 5614, 5528, 5480] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280225792 dispatch_meta_bytes=228904 combine_payload_bytes=280225792 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280454696 ingress_bytes=610527232 link_bytes=610527232 token_bytes=560680488 weight_movement_bytes=330301440 total_nvlink_bytes=890981928 weight_to_token_ratio=0.589108 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061807271e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44330 payload_rows_remote=38738 dup_rows=21206 dup_rows_remote=18578 dedup_ratio=0.323578 router_tokens_per_expert=[509, 534, 482, 504, 515, 474, 514, 542, 506, 525, 543, 490, 503, 502, 491, 501, 536, 527, 491, 492, 507, 539, 517, 501, 496, 503, 500, 532, 494, 518, 492, 535, 521, 497, 524, 539, 517, 515, 470, 492, 550, 532, 549, 552, 490, 515, 486, 512, 516, 499, 550, 521, 489, 548, 492, 523, 501, 500, 511, 453, 527, 532, 519, 511] payload_rows_per_dst=[5584, 5538, 5592, 5470, 5524, 5614, 5528, 5480] dup_rows_per_dst=[2564, 2584, 2628, 2670, 2626, 2758, 2748, 2628] combine_rows_per_src=[5584, 5538, 5592, 5470, 5524, 5614, 5528, 5480] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44580 payload_rows_remote=39052 dup_rows=20956 dup_rows_remote=18442 dedup_ratio=0.319763 router_tokens_per_expert=[513, 485, 525, 515, 519, 508, 484, 519, 514, 492, 520, 470, 507, 488, 533, 497, 504, 502, 545, 524, 480, 518, 523, 522, 582, 514, 526, 511, 501, 495, 536, 548, 514, 491, 551, 522, 517, 490, 516, 495, 510, 480, 518, 506, 528, 535, 520, 501, 472, 509, 484, 520, 516, 491, 511, 516, 516, 523, 513, 522, 514, 512, 488, 547] payload_rows_per_dst=[5524, 5528, 5618, 5660, 5578, 5562, 5482, 5628] dup_rows_per_dst=[2612, 2514, 2618, 2766, 2614, 2634, 2556, 2642] combine_rows_per_src=[5524, 5528, 5618, 5660, 5578, 5562, 5482, 5628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278376448 dispatch_meta_bytes=228864 combine_payload_bytes=278376448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278605312 ingress_bytes=388476928 link_bytes=388476928 token_bytes=556981760 weight_movement_bytes=110100480 total_nvlink_bytes=667082240 weight_to_token_ratio=0.197673 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756252713e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=229192 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279623496 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559017800 weight_movement_bytes=110100480 total_nvlink_bytes=669118280 weight_to_token_ratio=0.196953 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=38982 dup_rows=21024 dup_rows_remote=18322 dedup_ratio=0.320801 router_tokens_per_expert=[513, 552, 521, 469, 515, 508, 515, 472, 496, 507, 503, 496, 529, 526, 485, 523, 503, 472, 514, 509, 542, 495, 489, 551, 503, 539, 472, 564, 580, 519, 513, 543, 492, 486, 518, 531, 501, 518, 496, 526, 494, 538, 499, 487, 469, 558, 558, 481, 504, 520, 498, 505, 534, 534, 522, 499, 500, 530, 510, 498, 501, 523, 493, 507] payload_rows_per_dst=[5576, 5546, 5520, 5734, 5572, 5532, 5530, 5502] dup_rows_per_dst=[2554, 2584, 2630, 2732, 2564, 2636, 2702, 2622] combine_rows_per_src=[5576, 5546, 5520, 5734, 5572, 5532, 5530, 5502] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=228344 combine_payload_bytes=278763520 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278991864 ingress_bytes=609064960 link_bytes=609064960 token_bytes=557755384 weight_movement_bytes=330301440 total_nvlink_bytes=888056824 weight_to_token_ratio=0.592198 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059264139e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=39008 dup_rows=20950 dup_rows_remote=18364 dedup_ratio=0.319672 router_tokens_per_expert=[489, 503, 500, 519, 535, 525, 508, 530, 523, 497, 510, 513, 507, 497, 522, 513, 509, 529, 508, 564, 535, 510, 504, 498, 497, 507, 519, 525, 497, 532, 508, 511, 468, 504, 539, 501, 499, 470, 497, 519, 492, 500, 536, 508, 481, 497, 527, 500, 516, 493, 488, 506, 508, 528, 494, 522, 533, 493, 551, 530, 515, 541, 538, 530] payload_rows_per_dst=[5578, 5578, 5602, 5608, 5452, 5534, 5568, 5666] dup_rows_per_dst=[2640, 2586, 2712, 2584, 2542, 2548, 2542, 2796] combine_rows_per_src=[5578, 5578, 5602, 5608, 5452, 5534, 5568, 5666] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44440 payload_rows_remote=39000 dup_rows=21096 dup_rows_remote=18516 dedup_ratio=0.321899 router_tokens_per_expert=[537, 498, 498, 517, 554, 476, 495, 504, 528, 505, 482, 557, 518, 516, 500, 525, 497, 559, 526, 509, 516, 515, 472, 510, 522, 514, 491, 509, 558, 511, 534, 513, 500, 516, 504, 479, 509, 504, 480, 518, 486, 545, 501, 547, 477, 502, 501, 498, 503, 511, 544, 532, 514, 515, 513, 509, 491, 520, 535, 521, 467, 492, 527, 541] payload_rows_per_dst=[5576, 5574, 5582, 5588, 5440, 5502, 5604, 5574] dup_rows_per_dst=[2582, 2688, 2626, 2716, 2580, 2612, 2678, 2614] combine_rows_per_src=[5576, 5574, 5582, 5588, 5440, 5502, 5604, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279924736 dispatch_meta_bytes=229976 combine_payload_bytes=279924736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280154712 ingress_bytes=390025216 link_bytes=390025216 token_bytes=560079448 weight_movement_bytes=110100480 total_nvlink_bytes=670179928 weight_to_token_ratio=0.196580 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.783179987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=229192 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279623496 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559017800 weight_movement_bytes=110100480 total_nvlink_bytes=669118280 weight_to_token_ratio=0.196953 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277673984 dispatch_meta_bytes=229264 combine_payload_bytes=277673984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277903248 ingress_bytes=387774464 link_bytes=387774464 token_bytes=555577232 weight_movement_bytes=110100480 total_nvlink_bytes=665677712 weight_to_token_ratio=0.198173 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744035708e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38956 dup_rows=20978 dup_rows_remote=18470 dedup_ratio=0.320099 router_tokens_per_expert=[501, 527, 505, 533, 495, 482, 524, 504, 507, 509, 504, 489, 522, 482, 505, 553, 500, 508, 521, 550, 496, 529, 549, 506, 494, 543, 451, 509, 539, 512, 513, 494, 519, 520, 537, 532, 533, 509, 568, 500, 489, 518, 478, 508, 515, 486, 505, 510, 508, 548, 519, 529, 505, 480, 505, 497, 530, 500, 503, 510, 513, 519, 518, 501] payload_rows_per_dst=[5520, 5544, 5640, 5602, 5696, 5444, 5570, 5542] dup_rows_per_dst=[2622, 2598, 2678, 2508, 2740, 2574, 2612, 2646] combine_rows_per_src=[5520, 5544, 5640, 5602, 5696, 5444, 5570, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279609344 dispatch_meta_bytes=229488 combine_payload_bytes=279609344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279838832 ingress_bytes=389709824 link_bytes=389709824 token_bytes=559448176 weight_movement_bytes=110100480 total_nvlink_bytes=669548656 weight_to_token_ratio=0.196802 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777694802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=38882 dup_rows=20968 dup_rows_remote=18358 dedup_ratio=0.319946 router_tokens_per_expert=[518, 500, 548, 542, 497, 466, 514, 507, 539, 521, 535, 471, 507, 509, 505, 541, 517, 523, 493, 523, 507, 503, 531, 501, 492, 529, 531, 488, 465, 521, 483, 517, 487, 503, 547, 527, 494, 523, 511, 536, 508, 498, 521, 538, 519, 479, 501, 484, 560, 514, 509, 472, 507, 483, 537, 520, 522, 486, 498, 477, 560, 526, 525, 552] payload_rows_per_dst=[5588, 5578, 5612, 5494, 5540, 5560, 5510, 5686] dup_rows_per_dst=[2596, 2678, 2548, 2590, 2716, 2536, 2694, 2610] combine_rows_per_src=[5588, 5578, 5612, 5494, 5540, 5560, 5510, 5686] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229216 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279652192 ingress_bytes=499623936 link_bytes=499623936 token_bytes=559075168 weight_movement_bytes=220200960 total_nvlink_bytes=779276128 weight_to_token_ratio=0.393866 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689281987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44440 payload_rows_remote=39000 dup_rows=21096 dup_rows_remote=18516 dedup_ratio=0.321899 router_tokens_per_expert=[537, 498, 498, 517, 554, 476, 495, 504, 528, 505, 482, 557, 518, 516, 500, 525, 497, 559, 526, 509, 516, 515, 472, 510, 522, 514, 491, 509, 558, 511, 534, 513, 500, 516, 504, 479, 509, 504, 480, 518, 486, 545, 501, 547, 477, 502, 501, 498, 503, 511, 544, 532, 514, 515, 513, 509, 491, 520, 535, 521, 467, 492, 527, 541] payload_rows_per_dst=[5576, 5574, 5582, 5588, 5440, 5502, 5604, 5574] dup_rows_per_dst=[2582, 2688, 2626, 2716, 2580, 2612, 2678, 2614] combine_rows_per_src=[5576, 5574, 5582, 5588, 5440, 5502, 5604, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229704 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279466312 ingress_bytes=499437568 link_bytes=499437568 token_bytes=558702920 weight_movement_bytes=220200960 total_nvlink_bytes=778903880 weight_to_token_ratio=0.394129 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686040741e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44542 payload_rows_remote=38914 dup_rows=20994 dup_rows_remote=18278 dedup_ratio=0.320343 router_tokens_per_expert=[520, 476, 536, 497, 525, 492, 482, 529, 513, 504, 526, 492, 498, 490, 503, 496, 525, 509, 549, 513, 528, 508, 564, 495, 505, 511, 526, 518, 530, 496, 512, 496, 500, 510, 483, 501, 485, 525, 513, 481, 517, 511, 533, 506, 489, 523, 546, 547, 531, 504, 546, 494, 510, 489, 478, 546, 528, 524, 520, 522, 532, 515, 500, 495] payload_rows_per_dst=[5544, 5524, 5632, 5556, 5470, 5628, 5580, 5608] dup_rows_per_dst=[2570, 2520, 2750, 2632, 2526, 2716, 2616, 2664] combine_rows_per_src=[5544, 5524, 5632, 5556, 5470, 5628, 5580, 5608] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=230064 combine_payload_bytes=279552000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279782064 ingress_bytes=389652480 link_bytes=389652480 token_bytes=559334064 weight_movement_bytes=110100480 total_nvlink_bytes=669434544 weight_to_token_ratio=0.196842 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776697495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44512 payload_rows_remote=38982 dup_rows=21024 dup_rows_remote=18322 dedup_ratio=0.320801 router_tokens_per_expert=[513, 552, 521, 469, 515, 508, 515, 472, 496, 507, 503, 496, 529, 526, 485, 523, 503, 472, 514, 509, 542, 495, 489, 551, 503, 539, 472, 564, 580, 519, 513, 543, 492, 486, 518, 531, 501, 518, 496, 526, 494, 538, 499, 487, 469, 558, 558, 481, 504, 520, 498, 505, 534, 534, 522, 499, 500, 530, 510, 498, 501, 523, 493, 507] payload_rows_per_dst=[5576, 5546, 5520, 5734, 5572, 5532, 5530, 5502] dup_rows_per_dst=[2554, 2584, 2630, 2732, 2564, 2636, 2702, 2622] combine_rows_per_src=[5576, 5546, 5520, 5734, 5572, 5532, 5530, 5502] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38956 dup_rows=20978 dup_rows_remote=18470 dedup_ratio=0.320099 router_tokens_per_expert=[501, 527, 505, 533, 495, 482, 524, 504, 507, 509, 504, 489, 522, 482, 505, 553, 500, 508, 521, 550, 496, 529, 549, 506, 494, 543, 451, 509, 539, 512, 513, 494, 519, 520, 537, 532, 533, 509, 568, 500, 489, 518, 478, 508, 515, 486, 505, 510, 508, 548, 519, 529, 505, 480, 505, 497, 530, 500, 503, 510, 513, 519, 518, 501] payload_rows_per_dst=[5520, 5544, 5640, 5602, 5696, 5444, 5570, 5542] dup_rows_per_dst=[2622, 2598, 2678, 2508, 2740, 2574, 2612, 2646] combine_rows_per_src=[5520, 5544, 5640, 5602, 5696, 5444, 5570, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38856 dup_rows=21158 dup_rows_remote=18472 dedup_ratio=0.322845 router_tokens_per_expert=[501, 457, 536, 513, 512, 493, 533, 495, 492, 477, 549, 524, 511, 519, 510, 517, 518, 524, 505, 487, 533, 512, 512, 481, 505, 521, 488, 540, 543, 504, 516, 515, 491, 526, 504, 488, 504, 504, 493, 547, 504, 513, 531, 507, 510, 495, 510, 516, 520, 512, 507, 499, 539, 557, 495, 516, 533, 530, 537, 491, 524, 514, 486, 522] payload_rows_per_dst=[5522, 5746, 5468, 5336, 5616, 5612, 5476, 5602] dup_rows_per_dst=[2686, 2918, 2548, 2462, 2724, 2676, 2618, 2526] combine_rows_per_src=[5522, 5746, 5468, 5336, 5616, 5612, 5476, 5602] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=228960 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278935136 ingress_bytes=609007616 link_bytes=609007616 token_bytes=557641312 weight_movement_bytes=330301440 total_nvlink_bytes=887942752 weight_to_token_ratio=0.592319 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059164409e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=230064 combine_payload_bytes=279552000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279782064 ingress_bytes=389652480 link_bytes=389652480 token_bytes=559334064 weight_movement_bytes=110100480 total_nvlink_bytes=669434544 weight_to_token_ratio=0.196842 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776697495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38856 dup_rows=21158 dup_rows_remote=18472 dedup_ratio=0.322845 router_tokens_per_expert=[501, 457, 536, 513, 512, 493, 533, 495, 492, 477, 549, 524, 511, 519, 510, 517, 518, 524, 505, 487, 533, 512, 512, 481, 505, 521, 488, 540, 543, 504, 516, 515, 491, 526, 504, 488, 504, 504, 493, 547, 504, 513, 531, 507, 510, 495, 510, 516, 520, 512, 507, 499, 539, 557, 495, 516, 533, 530, 537, 491, 524, 514, 486, 522] payload_rows_per_dst=[5522, 5746, 5468, 5336, 5616, 5612, 5476, 5602] dup_rows_per_dst=[2686, 2918, 2548, 2462, 2724, 2676, 2618, 2526] combine_rows_per_src=[5522, 5746, 5468, 5336, 5616, 5612, 5476, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=228768 combine_payload_bytes=278935552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279164320 ingress_bytes=389036032 link_bytes=389036032 token_bytes=558099872 weight_movement_bytes=110100480 total_nvlink_bytes=668200352 weight_to_token_ratio=0.197277 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765976451e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=38828 dup_rows=21110 dup_rows_remote=18468 dedup_ratio=0.322113 router_tokens_per_expert=[500, 510, 514, 485, 513, 498, 516, 510, 498, 561, 536, 464, 509, 514, 535, 503, 526, 501, 488, 493, 547, 499, 517, 562, 475, 501, 543, 481, 534, 539, 499, 531, 527, 476, 486, 535, 504, 534, 491, 526, 488, 513, 496, 506, 500, 544, 534, 492, 517, 505, 528, 547, 537, 490, 532, 533, 501, 518, 540, 502, 463, 513, 496, 492] payload_rows_per_dst=[5506, 5598, 5558, 5540, 5552, 5520, 5698, 5454] dup_rows_per_dst=[2586, 2642, 2708, 2666, 2606, 2626, 2680, 2596] combine_rows_per_src=[5506, 5598, 5558, 5540, 5552, 5520, 5698, 5454] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44498 payload_rows_remote=38984 dup_rows=21038 dup_rows_remote=18486 dedup_ratio=0.321014 router_tokens_per_expert=[541, 485, 544, 559, 507, 524, 552, 520, 529, 490, 487, 488, 527, 489, 497, 491, 467, 518, 500, 486, 545, 515, 496, 506, 492, 514, 539, 496, 519, 490, 520, 510, 524, 539, 497, 509, 514, 507, 519, 471, 488, 538, 559, 497, 511, 547, 522, 494, 523, 505, 520, 538, 504, 532, 518, 514, 503, 495, 511, 504, 491, 514, 512, 505] payload_rows_per_dst=[5676, 5498, 5514, 5602, 5516, 5662, 5576, 5454] dup_rows_per_dst=[2788, 2498, 2552, 2558, 2644, 2650, 2732, 2616] combine_rows_per_src=[5676, 5498, 5514, 5602, 5516, 5662, 5576, 5454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229704 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279466312 ingress_bytes=499437568 link_bytes=499437568 token_bytes=558702920 weight_movement_bytes=220200960 total_nvlink_bytes=778903880 weight_to_token_ratio=0.394129 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686040741e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38886 dup_rows=21146 dup_rows_remote=18484 dedup_ratio=0.322662 router_tokens_per_expert=[496, 527, 517, 461, 518, 506, 517, 507, 511, 497, 550, 505, 525, 515, 495, 516, 469, 518, 538, 494, 532, 490, 518, 504, 481, 490, 545, 531, 529, 502, 551, 510, 519, 504, 524, 514, 500, 501, 502, 549, 499, 539, 534, 558, 539, 469, 491, 532, 479, 540, 526, 494, 541, 500, 491, 512, 524, 519, 491, 516, 495, 491, 513, 497] payload_rows_per_dst=[5550, 5580, 5538, 5518, 5572, 5600, 5504, 5528] dup_rows_per_dst=[2548, 2648, 2588, 2760, 2654, 2722, 2662, 2564] combine_rows_per_src=[5550, 5580, 5538, 5518, 5572, 5600, 5504, 5528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278519808 dispatch_meta_bytes=229312 combine_payload_bytes=278519808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278749120 ingress_bytes=388620288 link_bytes=388620288 token_bytes=557268928 weight_movement_bytes=110100480 total_nvlink_bytes=667369408 weight_to_token_ratio=0.197572 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758745979e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44184 payload_rows_remote=38594 dup_rows=21352 dup_rows_remote=18702 dedup_ratio=0.325806 router_tokens_per_expert=[474, 550, 546, 547, 501, 516, 521, 485, 499, 537, 502, 505, 508, 518, 522, 529, 499, 556, 488, 512, 539, 515, 488, 511, 495, 517, 479, 547, 507, 522, 521, 512, 504, 503, 499, 529, 495, 514, 485, 476, 539, 549, 492, 551, 524, 471, 523, 499, 493, 525, 506, 532, 487, 507, 537, 526, 503, 483, 502, 534, 484, 492, 502, 534] payload_rows_per_dst=[5576, 5590, 5566, 5496, 5468, 5518, 5524, 5446] dup_rows_per_dst=[2704, 2650, 2650, 2704, 2542, 2778, 2702, 2622] combine_rows_per_src=[5576, 5590, 5566, 5496, 5468, 5518, 5524, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229216 combine_payload_bytes=279422976 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279652192 ingress_bytes=499623936 link_bytes=499623936 token_bytes=559075168 weight_movement_bytes=220200960 total_nvlink_bytes=779276128 weight_to_token_ratio=0.393866 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689281987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38898 dup_rows=21114 dup_rows_remote=18436 dedup_ratio=0.322174 router_tokens_per_expert=[504, 531, 493, 500, 548, 512, 489, 540, 491, 476, 505, 517, 534, 525, 532, 492, 519, 504, 535, 517, 485, 544, 498, 526, 527, 547, 491, 503, 535, 544, 449, 505, 510, 488, 503, 503, 502, 510, 511, 534, 511, 512, 551, 530, 502, 518, 477, 524, 517, 452, 491, 508, 496, 518, 535, 504, 490, 524, 528, 504, 544, 525, 558, 470] payload_rows_per_dst=[5508, 5536, 5600, 5524, 5506, 5614, 5490, 5644] dup_rows_per_dst=[2726, 2608, 2656, 2678, 2616, 2636, 2552, 2642] combine_rows_per_src=[5508, 5536, 5600, 5524, 5506, 5614, 5490, 5644] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38978 dup_rows=21002 dup_rows_remote=18292 dedup_ratio=0.320465 router_tokens_per_expert=[508, 491, 554, 521, 528, 471, 505, 511, 504, 520, 492, 535, 526, 473, 470, 515, 488, 511, 527, 511, 565, 513, 535, 482, 550, 492, 511, 533, 492, 501, 475, 515, 492, 489, 564, 513, 525, 516, 534, 500, 570, 534, 503, 500, 528, 475, 532, 484, 547, 487, 533, 490, 488, 495, 535, 511, 483, 536, 538, 539, 493, 499, 498, 512] payload_rows_per_dst=[5536, 5586, 5620, 5534, 5556, 5560, 5556, 5586] dup_rows_per_dst=[2642, 2484, 2644, 2604, 2710, 2692, 2616, 2610] combine_rows_per_src=[5536, 5586, 5620, 5534, 5556, 5560, 5556, 5586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278319104 dispatch_meta_bytes=229184 combine_payload_bytes=278319104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278548288 ingress_bytes=388419584 link_bytes=388419584 token_bytes=556867392 weight_movement_bytes=110100480 total_nvlink_bytes=666967872 weight_to_token_ratio=0.197714 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755255406e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278519808 dispatch_meta_bytes=229312 combine_payload_bytes=278519808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278749120 ingress_bytes=388620288 link_bytes=388620288 token_bytes=557268928 weight_movement_bytes=110100480 total_nvlink_bytes=667369408 weight_to_token_ratio=0.197572 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758745979e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=229880 combine_payload_bytes=279437312 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279667192 ingress_bytes=609738752 link_bytes=609738752 token_bytes=559104504 weight_movement_bytes=330301440 total_nvlink_bytes=889405944 weight_to_token_ratio=0.590769 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060435975e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=229480 combine_payload_bytes=278734848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278964328 ingress_bytes=388835328 link_bytes=388835328 token_bytes=557699176 weight_movement_bytes=110100480 total_nvlink_bytes=667799656 weight_to_token_ratio=0.197419 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762485878e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276641792 dispatch_meta_bytes=229184 combine_payload_bytes=276641792 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=276870976 ingress_bytes=606943232 link_bytes=606943232 token_bytes=553512768 weight_movement_bytes=330301440 total_nvlink_bytes=883814208 weight_to_token_ratio=0.596737 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.055574105e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38830 dup_rows=21090 dup_rows_remote=18336 dedup_ratio=0.321808 router_tokens_per_expert=[492, 502, 501, 531, 514, 526, 528, 505, 520, 482, 480, 493, 526, 536, 475, 520, 498, 507, 504, 535, 543, 482, 510, 513, 499, 548, 523, 506, 475, 487, 503, 532, 498, 541, 516, 505, 512, 552, 538, 500, 486, 482, 505, 491, 545, 535, 511, 519, 538, 552, 491, 538, 468, 489, 480, 497, 553, 512, 504, 520, 552, 499, 539, 504] payload_rows_per_dst=[5494, 5504, 5498, 5486, 5674, 5602, 5572, 5616] dup_rows_per_dst=[2704, 2560, 2686, 2514, 2650, 2546, 2676, 2754] combine_rows_per_src=[5494, 5504, 5498, 5486, 5674, 5602, 5572, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278820864 dispatch_meta_bytes=229336 combine_payload_bytes=278820864 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279050200 ingress_bytes=499021824 link_bytes=499021824 token_bytes=557871064 weight_movement_bytes=220200960 total_nvlink_bytes=778072024 weight_to_token_ratio=0.394717 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678810269e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38978 dup_rows=21002 dup_rows_remote=18292 dedup_ratio=0.320465 router_tokens_per_expert=[508, 491, 554, 521, 528, 471, 505, 511, 504, 520, 492, 535, 526, 473, 470, 515, 488, 511, 527, 511, 565, 513, 535, 482, 550, 492, 511, 533, 492, 501, 475, 515, 492, 489, 564, 513, 525, 516, 534, 500, 570, 534, 503, 500, 528, 475, 532, 484, 547, 487, 533, 490, 488, 495, 535, 511, 483, 536, 538, 539, 493, 499, 498, 512] payload_rows_per_dst=[5536, 5586, 5620, 5534, 5556, 5560, 5556, 5586] dup_rows_per_dst=[2642, 2484, 2644, 2604, 2710, 2692, 2616, 2610] combine_rows_per_src=[5536, 5586, 5620, 5534, 5556, 5560, 5556, 5586] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44534 payload_rows_remote=38918 dup_rows=21002 dup_rows_remote=18258 dedup_ratio=0.320465 router_tokens_per_expert=[510, 520, 492, 521, 508, 469, 494, 536, 521, 511, 512, 516, 482, 545, 529, 540, 486, 534, 517, 521, 484, 480, 553, 511, 536, 482, 487, 534, 500, 514, 517, 510, 495, 493, 539, 517, 500, 540, 525, 494, 534, 542, 523, 544, 534, 513, 508, 482, 514, 551, 508, 501, 505, 497, 520, 511, 473, 533, 510, 523, 500, 492, 487, 488] payload_rows_per_dst=[5566, 5634, 5504, 5550, 5522, 5616, 5624, 5518] dup_rows_per_dst=[2534, 2678, 2668, 2610, 2684, 2744, 2590, 2494] combine_rows_per_src=[5566, 5634, 5504, 5550, 5522, 5616, 5624, 5518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=229080 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279623384 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559017688 weight_movement_bytes=110100480 total_nvlink_bytes=669118168 weight_to_token_ratio=0.196953 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38898 dup_rows=21114 dup_rows_remote=18436 dedup_ratio=0.322174 router_tokens_per_expert=[504, 531, 493, 500, 548, 512, 489, 540, 491, 476, 505, 517, 534, 525, 532, 492, 519, 504, 535, 517, 485, 544, 498, 526, 527, 547, 491, 503, 535, 544, 449, 505, 510, 488, 503, 503, 502, 510, 511, 534, 511, 512, 551, 530, 502, 518, 477, 524, 517, 452, 491, 508, 496, 518, 535, 504, 490, 524, 528, 504, 544, 525, 558, 470] payload_rows_per_dst=[5508, 5536, 5600, 5524, 5506, 5614, 5490, 5644] dup_rows_per_dst=[2726, 2608, 2656, 2678, 2616, 2636, 2552, 2642] combine_rows_per_src=[5508, 5536, 5600, 5524, 5506, 5614, 5490, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38886 dup_rows=21146 dup_rows_remote=18484 dedup_ratio=0.322662 router_tokens_per_expert=[496, 527, 517, 461, 518, 506, 517, 507, 511, 497, 550, 505, 525, 515, 495, 516, 469, 518, 538, 494, 532, 490, 518, 504, 481, 490, 545, 531, 529, 502, 551, 510, 519, 504, 524, 514, 500, 501, 502, 549, 499, 539, 534, 558, 539, 469, 491, 532, 479, 540, 526, 494, 541, 500, 491, 512, 524, 519, 491, 516, 495, 491, 513, 497] payload_rows_per_dst=[5550, 5580, 5538, 5518, 5572, 5600, 5504, 5528] dup_rows_per_dst=[2548, 2648, 2588, 2760, 2654, 2722, 2662, 2564] combine_rows_per_src=[5550, 5580, 5538, 5518, 5572, 5600, 5504, 5528] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38820 dup_rows=21092 dup_rows_remote=18230 dedup_ratio=0.321838 router_tokens_per_expert=[493, 506, 518, 498, 488, 510, 528, 508, 548, 487, 519, 525, 510, 535, 510, 499, 529, 540, 455, 513, 507, 539, 517, 534, 488, 503, 555, 500, 495, 508, 528, 512, 507, 487, 494, 489, 501, 521, 481, 521, 507, 561, 530, 509, 499, 551, 514, 530, 511, 526, 489, 534, 513, 517, 488, 483, 486, 515, 544, 483, 520, 544, 512, 496] payload_rows_per_dst=[5624, 5690, 5496, 5560, 5482, 5462, 5554, 5576] dup_rows_per_dst=[2862, 2814, 2536, 2618, 2572, 2550, 2568, 2572] combine_rows_per_src=[5624, 5690, 5496, 5560, 5482, 5462, 5554, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44708 payload_rows_remote=39078 dup_rows=20828 dup_rows_remote=18222 dedup_ratio=0.317810 router_tokens_per_expert=[505, 485, 525, 549, 496, 505, 513, 497, 499, 486, 479, 491, 508, 515, 500, 509, 523, 509, 533, 502, 537, 491, 527, 496, 556, 515, 494, 555, 500, 504, 546, 517, 518, 489, 544, 517, 493, 513, 492, 524, 475, 516, 493, 512, 511, 491, 515, 510, 519, 498, 521, 532, 519, 485, 507, 516, 546, 487, 534, 521, 523, 557, 464, 559] payload_rows_per_dst=[5590, 5452, 5630, 5712, 5580, 5494, 5560, 5690] dup_rows_per_dst=[2560, 2522, 2606, 2662, 2600, 2552, 2634, 2692] combine_rows_per_src=[5590, 5452, 5630, 5712, 5580, 5494, 5560, 5690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44580 payload_rows_remote=39052 dup_rows=20956 dup_rows_remote=18442 dedup_ratio=0.319763 router_tokens_per_expert=[513, 485, 525, 515, 519, 508, 484, 519, 514, 492, 520, 470, 507, 488, 533, 497, 504, 502, 545, 524, 480, 518, 523, 522, 582, 514, 526, 511, 501, 495, 536, 548, 514, 491, 551, 522, 517, 490, 516, 495, 510, 480, 518, 506, 528, 535, 520, 501, 472, 509, 484, 520, 516, 491, 511, 516, 516, 523, 513, 522, 514, 512, 488, 547] payload_rows_per_dst=[5524, 5528, 5618, 5660, 5578, 5562, 5482, 5628] dup_rows_per_dst=[2612, 2514, 2618, 2766, 2614, 2634, 2556, 2642] combine_rows_per_src=[5524, 5528, 5618, 5660, 5578, 5562, 5482, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=228664 combine_payload_bytes=278333440 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278562104 ingress_bytes=498534400 link_bytes=498534400 token_bytes=556895544 weight_movement_bytes=220200960 total_nvlink_bytes=777096504 weight_to_token_ratio=0.395408 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670333164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=228704 combine_payload_bytes=278964224 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279192928 ingress_bytes=499165184 link_bytes=499165184 token_bytes=558157152 weight_movement_bytes=220200960 total_nvlink_bytes=778358112 weight_to_token_ratio=0.394514 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681303535e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38752 dup_rows=21134 dup_rows_remote=18494 dedup_ratio=0.322479 router_tokens_per_expert=[508, 485, 499, 522, 543, 521, 522, 537, 523, 516, 537, 495, 559, 530, 502, 483, 531, 528, 506, 501, 516, 512, 508, 492, 511, 491, 510, 518, 486, 496, 545, 494, 494, 510, 517, 536, 521, 473, 544, 511, 487, 497, 530, 493, 505, 518, 518, 514, 526, 513, 481, 532, 532, 492, 507, 518, 515, 516, 497, 495, 511, 526, 536, 476] payload_rows_per_dst=[5524, 5650, 5584, 5486, 5536, 5512, 5588, 5522] dup_rows_per_dst=[2750, 2640, 2604, 2616, 2676, 2612, 2614, 2622] combine_rows_per_src=[5524, 5650, 5584, 5486, 5536, 5512, 5588, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44708 payload_rows_remote=39078 dup_rows=20828 dup_rows_remote=18222 dedup_ratio=0.317810 router_tokens_per_expert=[505, 485, 525, 549, 496, 505, 513, 497, 499, 486, 479, 491, 508, 515, 500, 509, 523, 509, 533, 502, 537, 491, 527, 496, 556, 515, 494, 555, 500, 504, 546, 517, 518, 489, 544, 517, 493, 513, 492, 524, 475, 516, 493, 512, 511, 491, 515, 510, 519, 498, 521, 532, 519, 485, 507, 516, 546, 487, 534, 521, 523, 557, 464, 559] payload_rows_per_dst=[5590, 5452, 5630, 5712, 5580, 5494, 5560, 5690] dup_rows_per_dst=[2560, 2522, 2606, 2662, 2600, 2552, 2634, 2692] combine_rows_per_src=[5590, 5452, 5630, 5712, 5580, 5494, 5560, 5690] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=39044 dup_rows=21004 dup_rows_remote=18438 dedup_ratio=0.320496 router_tokens_per_expert=[550, 521, 516, 546, 495, 527, 497, 489, 511, 492, 515, 480, 487, 514, 538, 481, 540, 526, 552, 505, 534, 542, 504, 516, 539, 535, 518, 514, 530, 524, 507, 541, 480, 461, 483, 481, 502, 480, 516, 533, 504, 509, 547, 502, 524, 510, 499, 493, 467, 470, 490, 512, 545, 505, 531, 507, 521, 475, 477, 515, 539, 541, 539, 524] payload_rows_per_dst=[5594, 5528, 5656, 5678, 5404, 5586, 5488, 5598] dup_rows_per_dst=[2688, 2508, 2782, 2738, 2468, 2590, 2566, 2664] combine_rows_per_src=[5594, 5528, 5656, 5678, 5404, 5586, 5488, 5598] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278820864 dispatch_meta_bytes=229336 combine_payload_bytes=278820864 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279050200 ingress_bytes=499021824 link_bytes=499021824 token_bytes=557871064 weight_movement_bytes=220200960 total_nvlink_bytes=778072024 weight_to_token_ratio=0.394717 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678810269e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=228200 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278489960 ingress_bytes=388362240 link_bytes=388362240 token_bytes=556751720 weight_movement_bytes=110100480 total_nvlink_bytes=666852200 weight_to_token_ratio=0.197755 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754258100e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=39030 dup_rows=20990 dup_rows_remote=18442 dedup_ratio=0.320282 router_tokens_per_expert=[502, 487, 513, 567, 535, 516, 515, 541, 509, 496, 543, 519, 522, 505, 491, 515, 492, 502, 488, 497, 538, 562, 522, 519, 523, 522, 531, 543, 548, 491, 467, 491, 482, 502, 536, 486, 506, 496, 538, 486, 499, 530, 514, 517, 456, 488, 507, 530, 491, 526, 542, 527, 501, 521, 469, 527, 550, 498, 503, 485, 470, 548, 554, 471] payload_rows_per_dst=[5628, 5500, 5552, 5638, 5516, 5568, 5580, 5564] dup_rows_per_dst=[2724, 2700, 2688, 2594, 2548, 2514, 2628, 2594] combine_rows_per_src=[5628, 5500, 5552, 5638, 5516, 5568, 5580, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277774336 dispatch_meta_bytes=228984 combine_payload_bytes=277774336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278003320 ingress_bytes=387874816 link_bytes=387874816 token_bytes=555777656 weight_movement_bytes=110100480 total_nvlink_bytes=665878136 weight_to_token_ratio=0.198102 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745780995e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279867392 dispatch_meta_bytes=229928 combine_payload_bytes=279867392 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280097320 ingress_bytes=500068352 link_bytes=500068352 token_bytes=559964712 weight_movement_bytes=220200960 total_nvlink_bytes=780165672 weight_to_token_ratio=0.393241 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697011112e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=229080 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279623384 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559017688 weight_movement_bytes=110100480 total_nvlink_bytes=669118168 weight_to_token_ratio=0.196953 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=228704 combine_payload_bytes=278964224 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279192928 ingress_bytes=499165184 link_bytes=499165184 token_bytes=558157152 weight_movement_bytes=220200960 total_nvlink_bytes=778358112 weight_to_token_ratio=0.394514 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681303535e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=229480 combine_payload_bytes=278734848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278964328 ingress_bytes=388835328 link_bytes=388835328 token_bytes=557699176 weight_movement_bytes=110100480 total_nvlink_bytes=667799656 weight_to_token_ratio=0.197419 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762485878e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=228200 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278489960 ingress_bytes=388362240 link_bytes=388362240 token_bytes=556751720 weight_movement_bytes=110100480 total_nvlink_bytes=666852200 weight_to_token_ratio=0.197755 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754258100e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280111104 dispatch_meta_bytes=229200 combine_payload_bytes=280111104 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280340304 ingress_bytes=500312064 link_bytes=500312064 token_bytes=560451408 weight_movement_bytes=220200960 total_nvlink_bytes=780652368 weight_to_token_ratio=0.392899 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701249665e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279924736 dispatch_meta_bytes=229976 combine_payload_bytes=279924736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280154712 ingress_bytes=390025216 link_bytes=390025216 token_bytes=560079448 weight_movement_bytes=110100480 total_nvlink_bytes=670179928 weight_to_token_ratio=0.196580 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.783179987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44734 payload_rows_remote=39226 dup_rows=20802 dup_rows_remote=18254 dedup_ratio=0.317413 router_tokens_per_expert=[495, 510, 538, 482, 543, 506, 547, 519, 501, 500, 495, 489, 473, 512, 546, 506, 510, 494, 509, 503, 531, 498, 487, 513, 548, 474, 499, 518, 464, 523, 459, 543, 515, 503, 503, 539, 534, 527, 518, 499, 516, 520, 525, 504, 518, 542, 555, 464, 516, 510, 523, 516, 521, 527, 513, 515, 503, 553, 527, 518, 499, 509, 493, 508] payload_rows_per_dst=[5638, 5560, 5504, 5508, 5648, 5656, 5606, 5614] dup_rows_per_dst=[2642, 2484, 2586, 2548, 2628, 2632, 2676, 2606] combine_rows_per_src=[5638, 5560, 5504, 5508, 5648, 5656, 5606, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38978 dup_rows=21072 dup_rows_remote=18618 dedup_ratio=0.321533 router_tokens_per_expert=[508, 510, 531, 514, 534, 547, 531, 523, 503, 486, 514, 486, 512, 498, 499, 499, 535, 533, 517, 507, 528, 510, 509, 518, 496, 488, 481, 504, 526, 530, 533, 502, 504, 512, 505, 511, 465, 548, 492, 524, 525, 536, 514, 520, 482, 522, 540, 524, 520, 495, 490, 513, 520, 575, 543, 506, 517, 475, 497, 486, 468, 516, 474, 537] payload_rows_per_dst=[5364, 5518, 5370, 5672, 5700, 5724, 5630, 5486] dup_rows_per_dst=[2448, 2724, 2390, 2700, 2820, 2842, 2694, 2454] combine_rows_per_src=[5364, 5518, 5370, 5672, 5700, 5724, 5630, 5486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44434 payload_rows_remote=38904 dup_rows=21102 dup_rows_remote=18508 dedup_ratio=0.321991 router_tokens_per_expert=[497, 498, 506, 507, 492, 508, 522, 496, 481, 462, 499, 511, 505, 519, 499, 572, 560, 517, 521, 515, 531, 502, 495, 534, 524, 514, 447, 570, 526, 486, 495, 492, 534, 504, 523, 488, 508, 512, 527, 505, 494, 541, 521, 502, 504, 519, 469, 512, 509, 499, 516, 529, 531, 527, 476, 478, 540, 521, 542, 519, 570, 497, 521, 527] payload_rows_per_dst=[5466, 5526, 5644, 5496, 5582, 5530, 5498, 5692] dup_rows_per_dst=[2586, 2570, 2706, 2612, 2620, 2594, 2632, 2782] combine_rows_per_src=[5466, 5526, 5644, 5496, 5582, 5530, 5498, 5692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=229888 combine_payload_bytes=279767040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279996928 ingress_bytes=389867520 link_bytes=389867520 token_bytes=559763968 weight_movement_bytes=110100480 total_nvlink_bytes=669864448 weight_to_token_ratio=0.196691 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780437395e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=281171968 dispatch_meta_bytes=229920 combine_payload_bytes=281171968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281401888 ingress_bytes=391272448 link_bytes=391272448 token_bytes=562573856 weight_movement_bytes=110100480 total_nvlink_bytes=672674336 weight_to_token_ratio=0.195708 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.804871403e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=39030 dup_rows=20990 dup_rows_remote=18442 dedup_ratio=0.320282 router_tokens_per_expert=[502, 487, 513, 567, 535, 516, 515, 541, 509, 496, 543, 519, 522, 505, 491, 515, 492, 502, 488, 497, 538, 562, 522, 519, 523, 522, 531, 543, 548, 491, 467, 491, 482, 502, 536, 486, 506, 496, 538, 486, 499, 530, 514, 517, 456, 488, 507, 530, 491, 526, 542, 527, 501, 521, 469, 527, 550, 498, 503, 485, 470, 548, 554, 471] payload_rows_per_dst=[5628, 5500, 5552, 5638, 5516, 5568, 5580, 5564] dup_rows_per_dst=[2724, 2700, 2688, 2594, 2548, 2514, 2628, 2594] combine_rows_per_src=[5628, 5500, 5552, 5638, 5516, 5568, 5580, 5564] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44734 payload_rows_remote=39226 dup_rows=20802 dup_rows_remote=18254 dedup_ratio=0.317413 router_tokens_per_expert=[495, 510, 538, 482, 543, 506, 547, 519, 501, 500, 495, 489, 473, 512, 546, 506, 510, 494, 509, 503, 531, 498, 487, 513, 548, 474, 499, 518, 464, 523, 459, 543, 515, 503, 503, 539, 534, 527, 518, 499, 516, 520, 525, 504, 518, 542, 555, 464, 516, 510, 523, 516, 521, 527, 513, 515, 503, 553, 527, 518, 499, 509, 493, 508] payload_rows_per_dst=[5638, 5560, 5504, 5508, 5648, 5656, 5606, 5614] dup_rows_per_dst=[2642, 2484, 2586, 2548, 2628, 2632, 2676, 2606] combine_rows_per_src=[5638, 5560, 5504, 5508, 5648, 5656, 5606, 5614] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=230384 combine_payload_bytes=279394304 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279624688 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559018992 weight_movement_bytes=110100480 total_nvlink_bytes=669119472 weight_to_token_ratio=0.196953 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44434 payload_rows_remote=38904 dup_rows=21102 dup_rows_remote=18508 dedup_ratio=0.321991 router_tokens_per_expert=[497, 498, 506, 507, 492, 508, 522, 496, 481, 462, 499, 511, 505, 519, 499, 572, 560, 517, 521, 515, 531, 502, 495, 534, 524, 514, 447, 570, 526, 486, 495, 492, 534, 504, 523, 488, 508, 512, 527, 505, 494, 541, 521, 502, 504, 519, 469, 512, 509, 499, 516, 529, 531, 527, 476, 478, 540, 521, 542, 519, 570, 497, 521, 527] payload_rows_per_dst=[5466, 5526, 5644, 5496, 5582, 5530, 5498, 5692] dup_rows_per_dst=[2586, 2570, 2706, 2612, 2620, 2594, 2632, 2782] combine_rows_per_src=[5466, 5526, 5644, 5496, 5582, 5530, 5498, 5692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=39044 dup_rows=21004 dup_rows_remote=18438 dedup_ratio=0.320496 router_tokens_per_expert=[550, 521, 516, 546, 495, 527, 497, 489, 511, 492, 515, 480, 487, 514, 538, 481, 540, 526, 552, 505, 534, 542, 504, 516, 539, 535, 518, 514, 530, 524, 507, 541, 480, 461, 483, 481, 502, 480, 516, 533, 504, 509, 547, 502, 524, 510, 499, 493, 467, 470, 490, 512, 545, 505, 531, 507, 521, 475, 477, 515, 539, 541, 539, 524] payload_rows_per_dst=[5594, 5528, 5656, 5678, 5404, 5586, 5488, 5598] dup_rows_per_dst=[2688, 2508, 2782, 2738, 2468, 2590, 2566, 2664] combine_rows_per_src=[5594, 5528, 5656, 5678, 5404, 5586, 5488, 5598] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38748 dup_rows=21116 dup_rows_remote=18454 dedup_ratio=0.322205 router_tokens_per_expert=[510, 536, 485, 542, 536, 508, 499, 525, 483, 535, 540, 545, 519, 531, 516, 520, 494, 528, 478, 574, 489, 543, 520, 504, 499, 506, 492, 486, 521, 552, 506, 512, 524, 533, 484, 491, 516, 499, 491, 541, 517, 487, 529, 470, 496, 492, 489, 521, 527, 504, 525, 505, 514, 500, 526, 508, 501, 471, 531, 536, 467, 514, 496, 529] payload_rows_per_dst=[5672, 5578, 5596, 5514, 5536, 5468, 5532, 5524] dup_rows_per_dst=[2662, 2800, 2664, 2634, 2622, 2534, 2656, 2544] combine_rows_per_src=[5672, 5578, 5596, 5514, 5536, 5468, 5532, 5524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229648 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279093520 ingress_bytes=388964352 link_bytes=388964352 token_bytes=557957392 weight_movement_bytes=110100480 total_nvlink_bytes=668057872 weight_to_token_ratio=0.197328 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764729818e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38912 dup_rows=21076 dup_rows_remote=18408 dedup_ratio=0.321594 router_tokens_per_expert=[503, 469, 519, 513, 517, 479, 561, 512, 523, 520, 500, 502, 489, 485, 535, 554, 471, 542, 485, 530, 478, 498, 527, 526, 492, 519, 522, 509, 509, 521, 502, 548, 517, 496, 517, 519, 501, 527, 498, 531, 544, 505, 512, 518, 522, 490, 464, 525, 455, 526, 509, 535, 526, 528, 480, 543, 546, 501, 503, 517, 546, 518, 498, 491] payload_rows_per_dst=[5552, 5548, 5524, 5548, 5532, 5498, 5612, 5646] dup_rows_per_dst=[2594, 2668, 2590, 2696, 2680, 2662, 2592, 2594] combine_rows_per_src=[5552, 5548, 5524, 5548, 5532, 5498, 5612, 5646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44656 payload_rows_remote=39124 dup_rows=20880 dup_rows_remote=18342 dedup_ratio=0.318604 router_tokens_per_expert=[540, 495, 526, 547, 489, 499, 491, 511, 498, 489, 507, 514, 461, 485, 489, 540, 495, 497, 509, 516, 534, 524, 501, 536, 518, 544, 478, 524, 487, 501, 535, 495, 529, 502, 508, 497, 533, 509, 511, 525, 502, 519, 535, 546, 493, 547, 540, 512, 514, 517, 480, 500, 503, 489, 526, 506, 526, 527, 545, 493, 561, 484, 517, 497] payload_rows_per_dst=[5630, 5480, 5576, 5556, 5660, 5640, 5532, 5582] dup_rows_per_dst=[2566, 2486, 2648, 2608, 2568, 2748, 2538, 2718] combine_rows_per_src=[5630, 5480, 5576, 5556, 5660, 5640, 5532, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=39040 dup_rows=20948 dup_rows_remote=18286 dedup_ratio=0.319641 router_tokens_per_expert=[510, 513, 506, 492, 511, 528, 497, 518, 490, 495, 561, 534, 502, 526, 528, 489, 509, 513, 510, 475, 577, 517, 478, 526, 516, 509, 489, 486, 518, 476, 509, 548, 528, 487, 522, 521, 529, 511, 516, 502, 536, 501, 499, 486, 536, 536, 515, 478, 544, 506, 528, 495, 509, 493, 484, 515, 533, 515, 503, 539, 516, 541, 509, 479] payload_rows_per_dst=[5476, 5614, 5548, 5542, 5590, 5602, 5580, 5636] dup_rows_per_dst=[2674, 2636, 2662, 2560, 2642, 2572, 2568, 2634] combine_rows_per_src=[5476, 5614, 5548, 5542, 5590, 5602, 5580, 5636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=281171968 dispatch_meta_bytes=229920 combine_payload_bytes=281171968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281401888 ingress_bytes=391272448 link_bytes=391272448 token_bytes=562573856 weight_movement_bytes=110100480 total_nvlink_bytes=672674336 weight_to_token_ratio=0.195708 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.804871403e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277745664 dispatch_meta_bytes=228808 combine_payload_bytes=277745664 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=277974472 ingress_bytes=718147584 link_bytes=718147584 token_bytes=555720136 weight_movement_bytes=440401920 total_nvlink_bytes=996122056 weight_to_token_ratio=0.792489 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.248976764e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=230384 combine_payload_bytes=279394304 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279624688 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559018992 weight_movement_bytes=110100480 total_nvlink_bytes=669119472 weight_to_token_ratio=0.196953 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=38828 dup_rows=21110 dup_rows_remote=18468 dedup_ratio=0.322113 router_tokens_per_expert=[500, 510, 514, 485, 513, 498, 516, 510, 498, 561, 536, 464, 509, 514, 535, 503, 526, 501, 488, 493, 547, 499, 517, 562, 475, 501, 543, 481, 534, 539, 499, 531, 527, 476, 486, 535, 504, 534, 491, 526, 488, 513, 496, 506, 500, 544, 534, 492, 517, 505, 528, 547, 537, 490, 532, 533, 501, 518, 540, 502, 463, 513, 496, 492] payload_rows_per_dst=[5506, 5598, 5558, 5540, 5552, 5520, 5698, 5454] dup_rows_per_dst=[2586, 2642, 2708, 2666, 2606, 2626, 2680, 2596] combine_rows_per_src=[5506, 5598, 5558, 5540, 5552, 5520, 5698, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279767040 dispatch_meta_bytes=229888 combine_payload_bytes=279767040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279996928 ingress_bytes=389867520 link_bytes=389867520 token_bytes=559763968 weight_movement_bytes=110100480 total_nvlink_bytes=669864448 weight_to_token_ratio=0.196691 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780437395e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44576 payload_rows_remote=39040 dup_rows=20960 dup_rows_remote=18376 dedup_ratio=0.319824 router_tokens_per_expert=[520, 516, 512, 485, 474, 480, 521, 510, 528, 519, 507, 510, 535, 521, 505, 524, 547, 528, 526, 499, 519, 493, 518, 515, 517, 509, 507, 544, 531, 553, 499, 518, 475, 517, 518, 502, 541, 496, 504, 507, 507, 508, 458, 483, 496, 504, 505, 507, 502, 529, 532, 545, 515, 474, 480, 568, 482, 494, 523, 502, 554, 532, 510, 508] payload_rows_per_dst=[5486, 5630, 5652, 5598, 5536, 5478, 5608, 5588] dup_rows_per_dst=[2550, 2668, 2638, 2758, 2584, 2458, 2682, 2622] combine_rows_per_src=[5486, 5630, 5652, 5598, 5536, 5478, 5608, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=229280 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279150496 ingress_bytes=609222656 link_bytes=609222656 token_bytes=558071712 weight_movement_bytes=330301440 total_nvlink_bytes=888373152 weight_to_token_ratio=0.591862 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059538399e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280440832 dispatch_meta_bytes=229864 combine_payload_bytes=280440832 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280670696 ingress_bytes=500641792 link_bytes=500641792 token_bytes=561111528 weight_movement_bytes=220200960 total_nvlink_bytes=781312488 weight_to_token_ratio=0.392437 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.706984177e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=39028 dup_rows=20984 dup_rows_remote=18392 dedup_ratio=0.320190 router_tokens_per_expert=[479, 506, 507, 535, 486, 500, 505, 509, 556, 524, 504, 504, 466, 554, 544, 523, 489, 530, 509, 493, 539, 492, 514, 476, 492, 526, 524, 502, 515, 495, 534, 470, 523, 500, 500, 503, 511, 504, 505, 526, 497, 556, 532, 505, 518, 492, 553, 502, 486, 488, 518, 551, 497, 517, 518, 511, 530, 508, 543, 481, 536, 511, 541, 503] payload_rows_per_dst=[5446, 5696, 5540, 5524, 5582, 5620, 5520, 5624] dup_rows_per_dst=[2608, 2654, 2544, 2592, 2562, 2690, 2652, 2682] combine_rows_per_src=[5446, 5696, 5540, 5524, 5582, 5620, 5520, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229648 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279093520 ingress_bytes=388964352 link_bytes=388964352 token_bytes=557957392 weight_movement_bytes=110100480 total_nvlink_bytes=668057872 weight_to_token_ratio=0.197328 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764729818e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277745664 dispatch_meta_bytes=228808 combine_payload_bytes=277745664 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=277974472 ingress_bytes=718147584 link_bytes=718147584 token_bytes=555720136 weight_movement_bytes=440401920 total_nvlink_bytes=996122056 weight_to_token_ratio=0.792489 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.248976764e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279867392 dispatch_meta_bytes=229928 combine_payload_bytes=279867392 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280097320 ingress_bytes=500068352 link_bytes=500068352 token_bytes=559964712 weight_movement_bytes=220200960 total_nvlink_bytes=780165672 weight_to_token_ratio=0.393241 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697011112e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229304 combine_payload_bytes=279838720 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280068024 ingress_bytes=610140160 link_bytes=610140160 token_bytes=559906744 weight_movement_bytes=330301440 total_nvlink_bytes=890208184 weight_to_token_ratio=0.589922 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061134089e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278319104 dispatch_meta_bytes=229184 combine_payload_bytes=278319104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278548288 ingress_bytes=388419584 link_bytes=388419584 token_bytes=556867392 weight_movement_bytes=110100480 total_nvlink_bytes=666967872 weight_to_token_ratio=0.197714 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755255406e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=229680 combine_payload_bytes=279752704 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279982384 ingress_bytes=610054144 link_bytes=610054144 token_bytes=559735088 weight_movement_bytes=330301440 total_nvlink_bytes=890036528 weight_to_token_ratio=0.590103 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060984493e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38780 dup_rows=21166 dup_rows_remote=18534 dedup_ratio=0.322968 router_tokens_per_expert=[492, 531, 514, 474, 491, 501, 503, 488, 519, 498, 531, 553, 519, 525, 524, 536, 507, 500, 459, 485, 542, 497, 503, 505, 528, 516, 521, 523, 506, 472, 524, 529, 526, 509, 523, 483, 497, 486, 522, 481, 539, 548, 525, 509, 499, 491, 580, 508, 546, 500, 528, 534, 526, 540, 500, 533, 511, 507, 473, 511, 508, 491, 494, 524] payload_rows_per_dst=[5510, 5414, 5452, 5596, 5646, 5568, 5594, 5590] dup_rows_per_dst=[2478, 2494, 2504, 2654, 2726, 2830, 2848, 2632] combine_rows_per_src=[5510, 5414, 5452, 5596, 5646, 5568, 5594, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229664 combine_payload_bytes=279838720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280068384 ingress_bytes=389939200 link_bytes=389939200 token_bytes=559907104 weight_movement_bytes=110100480 total_nvlink_bytes=670007584 weight_to_token_ratio=0.196641 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781684028e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44576 payload_rows_remote=39040 dup_rows=20960 dup_rows_remote=18376 dedup_ratio=0.319824 router_tokens_per_expert=[520, 516, 512, 485, 474, 480, 521, 510, 528, 519, 507, 510, 535, 521, 505, 524, 547, 528, 526, 499, 519, 493, 518, 515, 517, 509, 507, 544, 531, 553, 499, 518, 475, 517, 518, 502, 541, 496, 504, 507, 507, 508, 458, 483, 496, 504, 505, 507, 502, 529, 532, 545, 515, 474, 480, 568, 482, 494, 523, 502, 554, 532, 510, 508] payload_rows_per_dst=[5486, 5630, 5652, 5598, 5536, 5478, 5608, 5588] dup_rows_per_dst=[2550, 2668, 2638, 2758, 2584, 2458, 2682, 2622] combine_rows_per_src=[5486, 5630, 5652, 5598, 5536, 5478, 5608, 5588] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44678 payload_rows_remote=39014 dup_rows=20858 dup_rows_remote=18000 dedup_ratio=0.318268 router_tokens_per_expert=[514, 522, 561, 498, 515, 487, 528, 527, 478, 536, 499, 517, 478, 527, 530, 534, 470, 524, 492, 536, 536, 530, 508, 507, 515, 511, 501, 510, 552, 496, 494, 464, 524, 514, 508, 526, 520, 491, 491, 490, 502, 551, 541, 539, 520, 515, 526, 510, 509, 525, 521, 502, 529, 534, 493, 503, 518, 461, 473, 532, 485, 487, 525, 506] payload_rows_per_dst=[5664, 5586, 5606, 5392, 5546, 5756, 5634, 5494] dup_rows_per_dst=[2858, 2674, 2600, 2414, 2582, 2770, 2480, 2480] combine_rows_per_src=[5664, 5586, 5606, 5392, 5546, 5756, 5634, 5494] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44514 payload_rows_remote=39124 dup_rows=21022 dup_rows_remote=18626 dedup_ratio=0.320770 router_tokens_per_expert=[485, 513, 532, 500, 531, 531, 572, 525, 486, 543, 523, 531, 534, 535, 489, 532, 491, 551, 515, 539, 538, 540, 470, 496, 519, 507, 510, 570, 532, 509, 509, 557, 529, 507, 513, 492, 537, 522, 456, 486, 452, 485, 481, 491, 487, 465, 525, 507, 471, 506, 487, 521, 531, 526, 534, 532, 514, 520, 497, 496, 502, 509, 489, 483] payload_rows_per_dst=[5646, 5648, 5600, 5652, 5510, 5390, 5558, 5510] dup_rows_per_dst=[2732, 2698, 2680, 2774, 2574, 2396, 2658, 2510] combine_rows_per_src=[5646, 5648, 5600, 5652, 5510, 5390, 5558, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=229256 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278204296 ingress_bytes=388075520 link_bytes=388075520 token_bytes=556179336 weight_movement_bytes=110100480 total_nvlink_bytes=666279816 weight_to_token_ratio=0.197959 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749271567e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44678 payload_rows_remote=39014 dup_rows=20858 dup_rows_remote=18000 dedup_ratio=0.318268 router_tokens_per_expert=[514, 522, 561, 498, 515, 487, 528, 527, 478, 536, 499, 517, 478, 527, 530, 534, 470, 524, 492, 536, 536, 530, 508, 507, 515, 511, 501, 510, 552, 496, 494, 464, 524, 514, 508, 526, 520, 491, 491, 490, 502, 551, 541, 539, 520, 515, 526, 510, 509, 525, 521, 502, 529, 534, 493, 503, 518, 461, 473, 532, 485, 487, 525, 506] payload_rows_per_dst=[5664, 5586, 5606, 5392, 5546, 5756, 5634, 5494] dup_rows_per_dst=[2858, 2674, 2600, 2414, 2582, 2770, 2480, 2480] combine_rows_per_src=[5664, 5586, 5606, 5392, 5546, 5756, 5634, 5494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=39028 dup_rows=20984 dup_rows_remote=18392 dedup_ratio=0.320190 router_tokens_per_expert=[479, 506, 507, 535, 486, 500, 505, 509, 556, 524, 504, 504, 466, 554, 544, 523, 489, 530, 509, 493, 539, 492, 514, 476, 492, 526, 524, 502, 515, 495, 534, 470, 523, 500, 500, 503, 511, 504, 505, 526, 497, 556, 532, 505, 518, 492, 553, 502, 486, 488, 518, 551, 497, 517, 518, 511, 530, 508, 543, 481, 536, 511, 541, 503] payload_rows_per_dst=[5446, 5696, 5540, 5524, 5582, 5620, 5520, 5624] dup_rows_per_dst=[2608, 2654, 2544, 2592, 2562, 2690, 2652, 2682] combine_rows_per_src=[5446, 5696, 5540, 5524, 5582, 5620, 5520, 5624] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280440832 dispatch_meta_bytes=231000 combine_payload_bytes=280440832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280671832 ingress_bytes=390541312 link_bytes=390541312 token_bytes=561112664 weight_movement_bytes=110100480 total_nvlink_bytes=671213144 weight_to_token_ratio=0.196218 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.792155746e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44780 payload_rows_remote=39168 dup_rows=20756 dup_rows_remote=18142 dedup_ratio=0.316711 router_tokens_per_expert=[501, 510, 501, 526, 498, 498, 553, 511, 504, 533, 523, 494, 565, 490, 504, 500, 519, 523, 526, 532, 458, 499, 516, 481, 496, 523, 523, 554, 500, 532, 482, 503, 522, 491, 500, 494, 514, 525, 521, 527, 492, 552, 509, 499, 527, 503, 509, 524, 542, 502, 527, 495, 480, 545, 496, 500, 524, 510, 560, 482, 516, 481, 500, 521] payload_rows_per_dst=[5610, 5612, 5554, 5606, 5544, 5646, 5614, 5594] dup_rows_per_dst=[2586, 2614, 2554, 2620, 2644, 2584, 2560, 2594] combine_rows_per_src=[5610, 5612, 5554, 5606, 5544, 5646, 5614, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44514 payload_rows_remote=38958 dup_rows=21022 dup_rows_remote=18344 dedup_ratio=0.320770 router_tokens_per_expert=[524, 500, 491, 480, 498, 508, 469, 510, 525, 541, 505, 515, 532, 525, 509, 480, 467, 498, 542, 537, 534, 496, 488, 531, 511, 514, 492, 560, 492, 521, 507, 528, 546, 489, 480, 530, 515, 556, 512, 516, 494, 522, 497, 528, 526, 505, 483, 502, 529, 495, 522, 525, 497, 502, 519, 528, 500, 554, 512, 532, 487, 509, 526, 500] payload_rows_per_dst=[5434, 5588, 5584, 5560, 5534, 5532, 5556, 5726] dup_rows_per_dst=[2526, 2676, 2602, 2690, 2754, 2582, 2678, 2514] combine_rows_per_src=[5434, 5588, 5584, 5560, 5534, 5532, 5556, 5726] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44656 payload_rows_remote=39124 dup_rows=20880 dup_rows_remote=18342 dedup_ratio=0.318604 router_tokens_per_expert=[540, 495, 526, 547, 489, 499, 491, 511, 498, 489, 507, 514, 461, 485, 489, 540, 495, 497, 509, 516, 534, 524, 501, 536, 518, 544, 478, 524, 487, 501, 535, 495, 529, 502, 508, 497, 533, 509, 511, 525, 502, 519, 535, 546, 493, 547, 540, 512, 514, 517, 480, 500, 503, 489, 526, 506, 526, 527, 545, 493, 561, 484, 517, 497] payload_rows_per_dst=[5630, 5480, 5576, 5556, 5660, 5640, 5532, 5582] dup_rows_per_dst=[2566, 2486, 2648, 2608, 2568, 2748, 2538, 2718] combine_rows_per_src=[5630, 5480, 5576, 5556, 5660, 5640, 5532, 5582] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44684 payload_rows_remote=39046 dup_rows=20852 dup_rows_remote=18108 dedup_ratio=0.318176 router_tokens_per_expert=[538, 564, 511, 458, 486, 532, 487, 532, 544, 549, 539, 448, 511, 523, 510, 472, 520, 512, 521, 504, 548, 520, 532, 534, 498, 512, 540, 451, 526, 527, 530, 503, 513, 544, 499, 524, 516, 521, 503, 504, 493, 500, 545, 470, 505, 471, 502, 530, 470, 480, 513, 533, 513, 486, 533, 486, 508, 500, 517, 518, 533, 523, 519, 514] payload_rows_per_dst=[5632, 5538, 5638, 5582, 5614, 5492, 5590, 5598] dup_rows_per_dst=[2584, 2654, 2744, 2592, 2634, 2540, 2438, 2666] combine_rows_per_src=[5632, 5538, 5638, 5582, 5614, 5492, 5590, 5598] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=228056 combine_payload_bytes=279652352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279880408 ingress_bytes=389752832 link_bytes=389752832 token_bytes=559532760 weight_movement_bytes=110100480 total_nvlink_bytes=669633240 weight_to_token_ratio=0.196772 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778442782e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=229680 combine_payload_bytes=279752704 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279982384 ingress_bytes=610054144 link_bytes=610054144 token_bytes=559735088 weight_movement_bytes=330301440 total_nvlink_bytes=890036528 weight_to_token_ratio=0.590103 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060984493e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=39074 dup_rows=20976 dup_rows_remote=18336 dedup_ratio=0.320068 router_tokens_per_expert=[493, 542, 491, 522, 451, 514, 526, 493, 482, 535, 516, 535, 531, 498, 506, 514, 511, 497, 547, 528, 493, 532, 502, 542, 458, 507, 492, 476, 527, 529, 495, 512, 518, 516, 527, 478, 520, 528, 488, 488, 488, 494, 522, 555, 555, 523, 483, 514, 492, 503, 552, 496, 499, 537, 494, 527, 549, 532, 501, 531, 539, 518, 510, 494] payload_rows_per_dst=[5500, 5566, 5622, 5434, 5486, 5658, 5610, 5684] dup_rows_per_dst=[2564, 2668, 2682, 2558, 2640, 2610, 2590, 2664] combine_rows_per_src=[5500, 5566, 5622, 5434, 5486, 5658, 5610, 5684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38752 dup_rows=21134 dup_rows_remote=18494 dedup_ratio=0.322479 router_tokens_per_expert=[508, 485, 499, 522, 543, 521, 522, 537, 523, 516, 537, 495, 559, 530, 502, 483, 531, 528, 506, 501, 516, 512, 508, 492, 511, 491, 510, 518, 486, 496, 545, 494, 494, 510, 517, 536, 521, 473, 544, 511, 487, 497, 530, 493, 505, 518, 518, 514, 526, 513, 481, 532, 532, 492, 507, 518, 515, 516, 497, 495, 511, 526, 536, 476] payload_rows_per_dst=[5524, 5650, 5584, 5486, 5536, 5512, 5588, 5522] dup_rows_per_dst=[2750, 2640, 2604, 2616, 2676, 2612, 2614, 2622] combine_rows_per_src=[5524, 5650, 5584, 5486, 5536, 5512, 5588, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229664 combine_payload_bytes=279838720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280068384 ingress_bytes=389939200 link_bytes=389939200 token_bytes=559907104 weight_movement_bytes=110100480 total_nvlink_bytes=670007584 weight_to_token_ratio=0.196641 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781684028e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44656 payload_rows_remote=39190 dup_rows=20880 dup_rows_remote=18260 dedup_ratio=0.318604 router_tokens_per_expert=[516, 513, 483, 518, 525, 536, 465, 527, 497, 477, 531, 537, 503, 539, 530, 492, 511, 488, 521, 520, 520, 515, 545, 543, 498, 513, 498, 500, 497, 514, 495, 528, 505, 525, 526, 501, 496, 535, 513, 531, 509, 545, 503, 531, 502, 521, 494, 497, 535, 503, 536, 490, 511, 500, 519, 495, 508, 506, 512, 492, 507, 528, 471, 526] payload_rows_per_dst=[5540, 5578, 5612, 5466, 5668, 5600, 5636, 5556] dup_rows_per_dst=[2626, 2634, 2714, 2620, 2596, 2604, 2542, 2544] combine_rows_per_src=[5540, 5578, 5612, 5466, 5668, 5600, 5636, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280756224 dispatch_meta_bytes=229240 combine_payload_bytes=280756224 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=280985464 ingress_bytes=390856704 link_bytes=390856704 token_bytes=561741688 weight_movement_bytes=110100480 total_nvlink_bytes=671842168 weight_to_token_ratio=0.195998 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.797640931e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=229208 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279480152 ingress_bytes=609552384 link_bytes=609552384 token_bytes=558731096 weight_movement_bytes=330301440 total_nvlink_bytes=889032536 weight_to_token_ratio=0.591164 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060111850e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44708 payload_rows_remote=39050 dup_rows=20828 dup_rows_remote=18080 dedup_ratio=0.317810 router_tokens_per_expert=[494, 503, 486, 521, 497, 502, 519, 515, 473, 512, 503, 519, 501, 539, 517, 480, 529, 501, 480, 549, 514, 461, 507, 522, 520, 513, 538, 561, 493, 542, 528, 512, 512, 476, 494, 528, 508, 539, 495, 480, 503, 524, 494, 514, 499, 518, 548, 478, 541, 510, 509, 525, 497, 535, 528, 536, 521, 534, 519, 505, 532, 480, 542, 493] payload_rows_per_dst=[5600, 5574, 5548, 5712, 5390, 5556, 5670, 5658] dup_rows_per_dst=[2474, 2506, 2578, 2702, 2414, 2692, 2714, 2748] combine_rows_per_src=[5600, 5574, 5548, 5712, 5390, 5556, 5670, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=228056 combine_payload_bytes=279652352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279880408 ingress_bytes=389752832 link_bytes=389752832 token_bytes=559532760 weight_movement_bytes=110100480 total_nvlink_bytes=669633240 weight_to_token_ratio=0.196772 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778442782e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280440832 dispatch_meta_bytes=231000 combine_payload_bytes=280440832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280671832 ingress_bytes=390541312 link_bytes=390541312 token_bytes=561112664 weight_movement_bytes=110100480 total_nvlink_bytes=671213144 weight_to_token_ratio=0.196218 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.792155746e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280440832 dispatch_meta_bytes=229864 combine_payload_bytes=280440832 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280670696 ingress_bytes=500641792 link_bytes=500641792 token_bytes=561111528 weight_movement_bytes=220200960 total_nvlink_bytes=781312488 weight_to_token_ratio=0.392437 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.706984177e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279881728 dispatch_meta_bytes=228616 combine_payload_bytes=279881728 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280110344 ingress_bytes=500082688 link_bytes=500082688 token_bytes=559992072 weight_movement_bytes=220200960 total_nvlink_bytes=780193032 weight_to_token_ratio=0.393222 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697260439e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280082432 dispatch_meta_bytes=229640 combine_payload_bytes=280082432 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=280312072 ingress_bytes=720484352 link_bytes=720484352 token_bytes=560394504 weight_movement_bytes=440401920 total_nvlink_bytes=1000796424 weight_to_token_ratio=0.785878 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.253040787e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277774336 dispatch_meta_bytes=228984 combine_payload_bytes=277774336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278003320 ingress_bytes=387874816 link_bytes=387874816 token_bytes=555777656 weight_movement_bytes=110100480 total_nvlink_bytes=665878136 weight_to_token_ratio=0.198102 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745780995e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280913920 dispatch_meta_bytes=229800 combine_payload_bytes=280913920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281143720 ingress_bytes=391014400 link_bytes=391014400 token_bytes=562057640 weight_movement_bytes=110100480 total_nvlink_bytes=672158120 weight_to_token_ratio=0.195888 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.800383524e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44708 payload_rows_remote=39050 dup_rows=20828 dup_rows_remote=18080 dedup_ratio=0.317810 router_tokens_per_expert=[494, 503, 486, 521, 497, 502, 519, 515, 473, 512, 503, 519, 501, 539, 517, 480, 529, 501, 480, 549, 514, 461, 507, 522, 520, 513, 538, 561, 493, 542, 528, 512, 512, 476, 494, 528, 508, 539, 495, 480, 503, 524, 494, 514, 499, 518, 548, 478, 541, 510, 509, 525, 497, 535, 528, 536, 521, 534, 519, 505, 532, 480, 542, 493] payload_rows_per_dst=[5600, 5574, 5548, 5712, 5390, 5556, 5670, 5658] dup_rows_per_dst=[2474, 2506, 2578, 2702, 2414, 2692, 2714, 2748] combine_rows_per_src=[5600, 5574, 5548, 5712, 5390, 5556, 5670, 5658] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38964 dup_rows=21010 dup_rows_remote=18382 dedup_ratio=0.320587 router_tokens_per_expert=[491, 546, 533, 512, 480, 531, 524, 534, 485, 484, 517, 526, 478, 490, 534, 504, 496, 501, 551, 532, 523, 517, 498, 514, 491, 536, 477, 509, 521, 489, 546, 494, 496, 503, 513, 509, 527, 484, 529, 535, 506, 536, 482, 493, 507, 500, 529, 542, 519, 513, 489, 519, 495, 516, 513, 478, 527, 530, 522, 521, 509, 511, 543, 508] payload_rows_per_dst=[5636, 5510, 5552, 5534, 5594, 5562, 5556, 5582] dup_rows_per_dst=[2666, 2526, 2712, 2592, 2598, 2628, 2528, 2760] combine_rows_per_src=[5636, 5510, 5552, 5534, 5594, 5562, 5556, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=39074 dup_rows=20976 dup_rows_remote=18336 dedup_ratio=0.320068 router_tokens_per_expert=[493, 542, 491, 522, 451, 514, 526, 493, 482, 535, 516, 535, 531, 498, 506, 514, 511, 497, 547, 528, 493, 532, 502, 542, 458, 507, 492, 476, 527, 529, 495, 512, 518, 516, 527, 478, 520, 528, 488, 488, 488, 494, 522, 555, 555, 523, 483, 514, 492, 503, 552, 496, 499, 537, 494, 527, 549, 532, 501, 531, 539, 518, 510, 494] payload_rows_per_dst=[5500, 5566, 5622, 5434, 5486, 5658, 5610, 5684] dup_rows_per_dst=[2564, 2668, 2682, 2558, 2640, 2610, 2590, 2664] combine_rows_per_src=[5500, 5566, 5622, 5434, 5486, 5658, 5610, 5684] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279910400 dispatch_meta_bytes=228520 combine_payload_bytes=279910400 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280138920 ingress_bytes=500111360 link_bytes=500111360 token_bytes=560049320 weight_movement_bytes=220200960 total_nvlink_bytes=780250280 weight_to_token_ratio=0.393181 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697759092e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38640 dup_rows=21108 dup_rows_remote=18084 dedup_ratio=0.322083 router_tokens_per_expert=[468, 506, 510, 541, 525, 497, 484, 538, 485, 508, 555, 519, 499, 542, 502, 535, 512, 508, 489, 506, 464, 507, 485, 523, 480, 475, 497, 542, 484, 482, 530, 490, 552, 516, 533, 529, 477, 522, 539, 483, 513, 525, 497, 504, 493, 533, 531, 494, 538, 505, 542, 525, 487, 499, 545, 515, 533, 511, 544, 552, 517, 519, 495, 512] payload_rows_per_dst=[5788, 5590, 5488, 5612, 5568, 5532, 5464, 5386] dup_rows_per_dst=[3024, 2700, 2584, 2582, 2734, 2648, 2530, 2306] combine_rows_per_src=[5788, 5590, 5488, 5612, 5568, 5532, 5464, 5386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44656 payload_rows_remote=39190 dup_rows=20880 dup_rows_remote=18260 dedup_ratio=0.318604 router_tokens_per_expert=[516, 513, 483, 518, 525, 536, 465, 527, 497, 477, 531, 537, 503, 539, 530, 492, 511, 488, 521, 520, 520, 515, 545, 543, 498, 513, 498, 500, 497, 514, 495, 528, 505, 525, 526, 501, 496, 535, 513, 531, 509, 545, 503, 531, 502, 521, 494, 497, 535, 503, 536, 490, 511, 500, 519, 495, 508, 506, 512, 492, 507, 528, 471, 526] payload_rows_per_dst=[5540, 5578, 5612, 5466, 5668, 5600, 5636, 5556] dup_rows_per_dst=[2626, 2634, 2714, 2620, 2596, 2604, 2542, 2544] combine_rows_per_src=[5540, 5578, 5612, 5466, 5668, 5600, 5636, 5556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38982 dup_rows=21068 dup_rows_remote=18476 dedup_ratio=0.321472 router_tokens_per_expert=[518, 539, 551, 488, 544, 506, 492, 513, 523, 541, 490, 489, 508, 512, 502, 474, 538, 547, 498, 516, 519, 517, 553, 489, 525, 518, 495, 514, 538, 508, 490, 509, 546, 526, 498, 482, 535, 554, 513, 534, 500, 492, 505, 500, 501, 485, 482, 521, 494, 525, 486, 503, 532, 492, 529, 519, 481, 522, 531, 507, 494, 524, 471, 520] payload_rows_per_dst=[5592, 5486, 5642, 5528, 5698, 5472, 5542, 5508] dup_rows_per_dst=[2710, 2592, 2712, 2666, 2678, 2500, 2618, 2592] combine_rows_per_src=[5592, 5486, 5642, 5528, 5698, 5472, 5542, 5508] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229384 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279523336 ingress_bytes=389394432 link_bytes=389394432 token_bytes=558817288 weight_movement_bytes=110100480 total_nvlink_bytes=668917768 weight_to_token_ratio=0.197024 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772209616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38904 dup_rows=21046 dup_rows_remote=18420 dedup_ratio=0.321136 router_tokens_per_expert=[509, 489, 537, 532, 524, 526, 514, 520, 509, 497, 531, 501, 530, 568, 516, 497, 533, 519, 520, 504, 481, 495, 512, 507, 497, 493, 510, 522, 530, 503, 475, 534, 499, 483, 535, 493, 522, 478, 543, 523, 509, 506, 487, 507, 528, 511, 516, 516, 499, 525, 500, 492, 507, 538, 526, 519, 549, 513, 506, 482, 513, 504, 498, 506] payload_rows_per_dst=[5620, 5550, 5538, 5568, 5536, 5514, 5586, 5578] dup_rows_per_dst=[2682, 2748, 2604, 2560, 2616, 2646, 2626, 2564] combine_rows_per_src=[5620, 5550, 5538, 5568, 5536, 5514, 5586, 5578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=276971520 dispatch_meta_bytes=226896 combine_payload_bytes=276971520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277198416 ingress_bytes=387072000 link_bytes=387072000 token_bytes=554169936 weight_movement_bytes=110100480 total_nvlink_bytes=664270416 weight_to_token_ratio=0.198676 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.731818704e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38964 dup_rows=21010 dup_rows_remote=18382 dedup_ratio=0.320587 router_tokens_per_expert=[491, 546, 533, 512, 480, 531, 524, 534, 485, 484, 517, 526, 478, 490, 534, 504, 496, 501, 551, 532, 523, 517, 498, 514, 491, 536, 477, 509, 521, 489, 546, 494, 496, 503, 513, 509, 527, 484, 529, 535, 506, 536, 482, 493, 507, 500, 529, 542, 519, 513, 489, 519, 495, 516, 513, 478, 527, 530, 522, 521, 509, 511, 543, 508] payload_rows_per_dst=[5636, 5510, 5552, 5534, 5594, 5562, 5556, 5582] dup_rows_per_dst=[2666, 2526, 2712, 2592, 2598, 2628, 2528, 2760] combine_rows_per_src=[5636, 5510, 5552, 5534, 5594, 5562, 5556, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44514 payload_rows_remote=38958 dup_rows=21022 dup_rows_remote=18344 dedup_ratio=0.320770 router_tokens_per_expert=[524, 500, 491, 480, 498, 508, 469, 510, 525, 541, 505, 515, 532, 525, 509, 480, 467, 498, 542, 537, 534, 496, 488, 531, 511, 514, 492, 560, 492, 521, 507, 528, 546, 489, 480, 530, 515, 556, 512, 516, 494, 522, 497, 528, 526, 505, 483, 502, 529, 495, 522, 525, 497, 502, 519, 528, 500, 554, 512, 532, 487, 509, 526, 500] payload_rows_per_dst=[5434, 5588, 5584, 5560, 5534, 5532, 5556, 5726] dup_rows_per_dst=[2526, 2676, 2602, 2690, 2754, 2582, 2678, 2514] combine_rows_per_src=[5434, 5588, 5584, 5560, 5534, 5532, 5556, 5726] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44292 payload_rows_remote=38796 dup_rows=21244 dup_rows_remote=18590 dedup_ratio=0.324158 router_tokens_per_expert=[495, 479, 502, 535, 487, 513, 482, 488, 535, 503, 523, 513, 501, 515, 513, 494, 475, 498, 512, 518, 517, 519, 504, 532, 496, 568, 501, 541, 513, 484, 517, 504, 516, 516, 516, 537, 562, 536, 484, 519, 517, 486, 515, 503, 486, 490, 552, 502, 526, 527, 530, 474, 524, 529, 509, 525, 567, 516, 481, 526, 526, 490, 502, 502] payload_rows_per_dst=[5394, 5506, 5496, 5594, 5626, 5524, 5582, 5570] dup_rows_per_dst=[2568, 2688, 2654, 2654, 2746, 2578, 2706, 2650] combine_rows_per_src=[5394, 5506, 5496, 5594, 5626, 5524, 5582, 5570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44196 payload_rows_remote=38610 dup_rows=21340 dup_rows_remote=18656 dedup_ratio=0.325623 router_tokens_per_expert=[544, 500, 500, 490, 494, 515, 516, 481, 497, 515, 524, 512, 551, 506, 516, 503, 540, 545, 483, 492, 508, 472, 552, 499, 503, 532, 522, 521, 508, 537, 488, 544, 529, 556, 528, 482, 517, 474, 519, 530, 513, 487, 488, 539, 484, 495, 512, 537, 499, 483, 508, 497, 533, 523, 495, 525, 480, 494, 510, 539, 515, 527, 533, 507] payload_rows_per_dst=[5446, 5522, 5524, 5596, 5586, 5486, 5512, 5524] dup_rows_per_dst=[2634, 2726, 2658, 2714, 2684, 2624, 2614, 2686] combine_rows_per_src=[5446, 5522, 5524, 5596, 5586, 5486, 5512, 5524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280913920 dispatch_meta_bytes=229800 combine_payload_bytes=280913920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281143720 ingress_bytes=391014400 link_bytes=391014400 token_bytes=562057640 weight_movement_bytes=110100480 total_nvlink_bytes=672158120 weight_to_token_ratio=0.195888 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.800383524e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280082432 dispatch_meta_bytes=229640 combine_payload_bytes=280082432 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=280312072 ingress_bytes=720484352 link_bytes=720484352 token_bytes=560394504 weight_movement_bytes=440401920 total_nvlink_bytes=1000796424 weight_to_token_ratio=0.785878 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.253040787e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38912 dup_rows=21076 dup_rows_remote=18408 dedup_ratio=0.321594 router_tokens_per_expert=[503, 469, 519, 513, 517, 479, 561, 512, 523, 520, 500, 502, 489, 485, 535, 554, 471, 542, 485, 530, 478, 498, 527, 526, 492, 519, 522, 509, 509, 521, 502, 548, 517, 496, 517, 519, 501, 527, 498, 531, 544, 505, 512, 518, 522, 490, 464, 525, 455, 526, 509, 535, 526, 528, 480, 543, 546, 501, 503, 517, 546, 518, 498, 491] payload_rows_per_dst=[5552, 5548, 5524, 5548, 5532, 5498, 5612, 5646] dup_rows_per_dst=[2594, 2668, 2590, 2696, 2680, 2662, 2592, 2594] combine_rows_per_src=[5552, 5548, 5524, 5548, 5532, 5498, 5612, 5646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38964 dup_rows=21124 dup_rows_remote=18478 dedup_ratio=0.322327 router_tokens_per_expert=[514, 516, 552, 506, 511, 508, 532, 485, 517, 514, 514, 512, 507, 507, 519, 536, 537, 537, 524, 506, 500, 502, 516, 485, 503, 513, 513, 512, 496, 482, 507, 521, 484, 539, 474, 522, 506, 496, 519, 500, 508, 517, 545, 486, 514, 536, 519, 517, 507, 517, 484, 530, 470, 506, 488, 549, 521, 504, 547, 485, 528, 547, 496, 503] payload_rows_per_dst=[5560, 5566, 5552, 5448, 5454, 5690, 5512, 5630] dup_rows_per_dst=[2688, 2686, 2662, 2646, 2626, 2594, 2590, 2632] combine_rows_per_src=[5560, 5566, 5552, 5448, 5454, 5690, 5512, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229832 combine_payload_bytes=279422976 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279652808 ingress_bytes=499623936 link_bytes=499623936 token_bytes=559075784 weight_movement_bytes=220200960 total_nvlink_bytes=779276744 weight_to_token_ratio=0.393866 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689281987e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229296 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279093168 ingress_bytes=388964352 link_bytes=388964352 token_bytes=557957040 weight_movement_bytes=110100480 total_nvlink_bytes=668057520 weight_to_token_ratio=0.197328 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764729818e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44438 payload_rows_remote=38780 dup_rows=21098 dup_rows_remote=18532 dedup_ratio=0.321930 router_tokens_per_expert=[553, 517, 534, 528, 512, 530, 507, 490, 502, 555, 515, 506, 533, 527, 535, 476, 481, 510, 560, 552, 516, 521, 534, 530, 530, 502, 485, 492, 536, 498, 488, 547, 503, 533, 496, 466, 503, 514, 518, 495, 524, 491, 504, 492, 524, 525, 507, 479, 493, 505, 521, 499, 517, 477, 492, 493, 551, 487, 473, 528, 537, 506, 523, 490] payload_rows_per_dst=[5404, 5578, 5512, 5558, 5506, 5686, 5536, 5658] dup_rows_per_dst=[2330, 2674, 2584, 2644, 2596, 2934, 2770, 2566] combine_rows_per_src=[5404, 5578, 5512, 5558, 5506, 5686, 5536, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=276971520 dispatch_meta_bytes=226896 combine_payload_bytes=276971520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277198416 ingress_bytes=387072000 link_bytes=387072000 token_bytes=554169936 weight_movement_bytes=110100480 total_nvlink_bytes=664270416 weight_to_token_ratio=0.198676 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.731818704e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=229208 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279480152 ingress_bytes=609552384 link_bytes=609552384 token_bytes=558731096 weight_movement_bytes=330301440 total_nvlink_bytes=889032536 weight_to_token_ratio=0.591164 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060111850e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229384 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279523336 ingress_bytes=389394432 link_bytes=389394432 token_bytes=558817288 weight_movement_bytes=110100480 total_nvlink_bytes=668917768 weight_to_token_ratio=0.197024 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772209616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278089728 dispatch_meta_bytes=229544 combine_payload_bytes=278089728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278319272 ingress_bytes=388190208 link_bytes=388190208 token_bytes=556409000 weight_movement_bytes=110100480 total_nvlink_bytes=666509480 weight_to_token_ratio=0.197877 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751266180e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276756480 dispatch_meta_bytes=229064 combine_payload_bytes=276756480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276985544 ingress_bytes=386856960 link_bytes=386856960 token_bytes=553742024 weight_movement_bytes=110100480 total_nvlink_bytes=663842504 weight_to_token_ratio=0.198830 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.728078805e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=229280 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279150496 ingress_bytes=609222656 link_bytes=609222656 token_bytes=558071712 weight_movement_bytes=330301440 total_nvlink_bytes=888373152 weight_to_token_ratio=0.591862 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059538399e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229768 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279523720 ingress_bytes=389394432 link_bytes=389394432 token_bytes=558817672 weight_movement_bytes=110100480 total_nvlink_bytes=668918152 weight_to_token_ratio=0.197024 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772209616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44438 payload_rows_remote=38780 dup_rows=21098 dup_rows_remote=18532 dedup_ratio=0.321930 router_tokens_per_expert=[553, 517, 534, 528, 512, 530, 507, 490, 502, 555, 515, 506, 533, 527, 535, 476, 481, 510, 560, 552, 516, 521, 534, 530, 530, 502, 485, 492, 536, 498, 488, 547, 503, 533, 496, 466, 503, 514, 518, 495, 524, 491, 504, 492, 524, 525, 507, 479, 493, 505, 521, 499, 517, 477, 492, 493, 551, 487, 473, 528, 537, 506, 523, 490] payload_rows_per_dst=[5404, 5578, 5512, 5558, 5506, 5686, 5536, 5658] dup_rows_per_dst=[2330, 2674, 2584, 2644, 2596, 2934, 2770, 2566] combine_rows_per_src=[5404, 5578, 5512, 5558, 5506, 5686, 5536, 5658] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44550 payload_rows_remote=39058 dup_rows=20986 dup_rows_remote=18452 dedup_ratio=0.320221 router_tokens_per_expert=[517, 527, 512, 550, 510, 521, 505, 494, 543, 492, 490, 503, 475, 529, 500, 505, 481, 525, 564, 508, 517, 489, 530, 475, 490, 496, 517, 546, 505, 539, 545, 527, 488, 510, 541, 529, 516, 498, 516, 503, 504, 502, 511, 481, 489, 508, 494, 524, 480, 502, 532, 513, 546, 535, 485, 529, 489, 496, 491, 542, 541, 533, 490, 523] payload_rows_per_dst=[5610, 5498, 5562, 5610, 5594, 5492, 5634, 5550] dup_rows_per_dst=[2662, 2576, 2616, 2720, 2608, 2534, 2610, 2660] combine_rows_per_src=[5610, 5498, 5562, 5610, 5594, 5492, 5634, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44196 payload_rows_remote=38610 dup_rows=21340 dup_rows_remote=18656 dedup_ratio=0.325623 router_tokens_per_expert=[544, 500, 500, 490, 494, 515, 516, 481, 497, 515, 524, 512, 551, 506, 516, 503, 540, 545, 483, 492, 508, 472, 552, 499, 503, 532, 522, 521, 508, 537, 488, 544, 529, 556, 528, 482, 517, 474, 519, 530, 513, 487, 488, 539, 484, 495, 512, 537, 499, 483, 508, 497, 533, 523, 495, 525, 480, 494, 510, 539, 515, 527, 533, 507] payload_rows_per_dst=[5446, 5522, 5524, 5596, 5586, 5486, 5512, 5524] dup_rows_per_dst=[2634, 2726, 2658, 2714, 2684, 2624, 2614, 2686] combine_rows_per_src=[5446, 5522, 5524, 5596, 5586, 5486, 5512, 5524] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=229248 combine_payload_bytes=277975040 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278204288 ingress_bytes=498176000 link_bytes=498176000 token_bytes=556179328 weight_movement_bytes=220200960 total_nvlink_bytes=776380288 weight_to_token_ratio=0.395917 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.664099999e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44642 payload_rows_remote=38978 dup_rows=20894 dup_rows_remote=18088 dedup_ratio=0.318817 router_tokens_per_expert=[571, 522, 540, 531, 533, 492, 495, 546, 537, 538, 536, 529, 503, 509, 518, 501, 537, 516, 515, 497, 491, 514, 491, 513, 496, 519, 488, 482, 518, 500, 501, 500, 486, 504, 512, 531, 499, 475, 502, 528, 514, 502, 514, 514, 497, 534, 513, 487, 513, 511, 501, 472, 540, 522, 511, 487, 518, 508, 514, 502, 517, 534, 487, 540] payload_rows_per_dst=[5664, 5668, 5632, 5546, 5462, 5562, 5584, 5524] dup_rows_per_dst=[2806, 2664, 2622, 2462, 2612, 2588, 2530, 2610] combine_rows_per_src=[5664, 5668, 5632, 5546, 5462, 5562, 5584, 5524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38904 dup_rows=21046 dup_rows_remote=18420 dedup_ratio=0.321136 router_tokens_per_expert=[509, 489, 537, 532, 524, 526, 514, 520, 509, 497, 531, 501, 530, 568, 516, 497, 533, 519, 520, 504, 481, 495, 512, 507, 497, 493, 510, 522, 530, 503, 475, 534, 499, 483, 535, 493, 522, 478, 543, 523, 509, 506, 487, 507, 528, 511, 516, 516, 499, 525, 500, 492, 507, 538, 526, 519, 549, 513, 506, 482, 513, 504, 498, 506] payload_rows_per_dst=[5620, 5550, 5538, 5568, 5536, 5514, 5586, 5578] dup_rows_per_dst=[2682, 2748, 2604, 2560, 2616, 2646, 2626, 2564] combine_rows_per_src=[5620, 5550, 5538, 5568, 5536, 5514, 5586, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44740 payload_rows_remote=39280 dup_rows=20796 dup_rows_remote=18306 dedup_ratio=0.317322 router_tokens_per_expert=[554, 527, 484, 524, 575, 522, 528, 513, 501, 511, 496, 477, 515, 508, 465, 502, 488, 488, 474, 474, 529, 523, 481, 531, 547, 512, 505, 517, 518, 491, 525, 526, 542, 553, 466, 519, 520, 502, 548, 469, 504, 524, 479, 479, 497, 514, 528, 498, 526, 508, 509, 541, 521, 535, 509, 530, 518, 524, 501, 501, 529, 502, 521, 520] payload_rows_per_dst=[5762, 5460, 5518, 5602, 5646, 5512, 5654, 5586] dup_rows_per_dst=[2692, 2490, 2458, 2680, 2592, 2534, 2704, 2646] combine_rows_per_src=[5762, 5460, 5518, 5602, 5646, 5512, 5654, 5586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38964 dup_rows=21124 dup_rows_remote=18478 dedup_ratio=0.322327 router_tokens_per_expert=[514, 516, 552, 506, 511, 508, 532, 485, 517, 514, 514, 512, 507, 507, 519, 536, 537, 537, 524, 506, 500, 502, 516, 485, 503, 513, 513, 512, 496, 482, 507, 521, 484, 539, 474, 522, 506, 496, 519, 500, 508, 517, 545, 486, 514, 536, 519, 517, 507, 517, 484, 530, 470, 506, 488, 549, 521, 504, 547, 485, 528, 547, 496, 503] payload_rows_per_dst=[5560, 5566, 5552, 5448, 5454, 5690, 5512, 5630] dup_rows_per_dst=[2688, 2686, 2662, 2646, 2626, 2594, 2590, 2632] combine_rows_per_src=[5560, 5566, 5552, 5448, 5454, 5690, 5512, 5630] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279967744 dispatch_meta_bytes=230040 combine_payload_bytes=279967744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280197784 ingress_bytes=390068224 link_bytes=390068224 token_bytes=560165528 weight_movement_bytes=110100480 total_nvlink_bytes=670266008 weight_to_token_ratio=0.196550 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.783927967e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38970 dup_rows=21026 dup_rows_remote=18324 dedup_ratio=0.320831 router_tokens_per_expert=[560, 481, 502, 487, 542, 512, 519, 515, 545, 543, 526, 509, 492, 486, 539, 514, 497, 498, 537, 473, 481, 504, 507, 479, 472, 493, 506, 555, 527, 526, 503, 504, 506, 523, 527, 553, 483, 518, 567, 488, 523, 523, 507, 522, 502, 505, 494, 518, 512, 530, 483, 546, 518, 501, 517, 514, 543, 509, 499, 489, 497, 515, 499, 503] payload_rows_per_dst=[5562, 5610, 5462, 5548, 5658, 5566, 5540, 5564] dup_rows_per_dst=[2674, 2698, 2490, 2624, 2672, 2622, 2702, 2544] combine_rows_per_src=[5562, 5610, 5462, 5548, 5658, 5566, 5540, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=228264 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279622568 ingress_bytes=719796224 link_bytes=719796224 token_bytes=559016872 weight_movement_bytes=440401920 total_nvlink_bytes=999418792 weight_to_token_ratio=0.787815 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251844020e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38838 dup_rows=21130 dup_rows_remote=18568 dedup_ratio=0.322418 router_tokens_per_expert=[520, 521, 539, 522, 545, 519, 502, 498, 492, 523, 484, 519, 506, 505, 505, 547, 541, 508, 496, 535, 543, 511, 505, 554, 518, 529, 514, 511, 460, 512, 513, 511, 513, 480, 495, 467, 554, 515, 494, 547, 515, 526, 521, 511, 520, 476, 492, 515, 539, 487, 490, 525, 487, 538, 580, 455, 496, 483, 515, 531, 511, 495, 477, 510] payload_rows_per_dst=[5608, 5526, 5628, 5492, 5568, 5652, 5494, 5438] dup_rows_per_dst=[2724, 2636, 2758, 2644, 2562, 2500, 2708, 2598] combine_rows_per_src=[5608, 5526, 5628, 5492, 5568, 5652, 5494, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44550 payload_rows_remote=39058 dup_rows=20986 dup_rows_remote=18452 dedup_ratio=0.320221 router_tokens_per_expert=[517, 527, 512, 550, 510, 521, 505, 494, 543, 492, 490, 503, 475, 529, 500, 505, 481, 525, 564, 508, 517, 489, 530, 475, 490, 496, 517, 546, 505, 539, 545, 527, 488, 510, 541, 529, 516, 498, 516, 503, 504, 502, 511, 481, 489, 508, 494, 524, 480, 502, 532, 513, 546, 535, 485, 529, 489, 496, 491, 542, 541, 533, 490, 523] payload_rows_per_dst=[5610, 5498, 5562, 5610, 5594, 5492, 5634, 5550] dup_rows_per_dst=[2662, 2576, 2616, 2720, 2608, 2534, 2610, 2660] combine_rows_per_src=[5610, 5498, 5562, 5610, 5594, 5492, 5634, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38938 dup_rows=21052 dup_rows_remote=18488 dedup_ratio=0.321228 router_tokens_per_expert=[499, 541, 505, 536, 522, 505, 537, 517, 530, 508, 559, 487, 503, 505, 499, 511, 491, 485, 499, 535, 537, 520, 483, 553, 508, 470, 529, 528, 537, 482, 487, 514, 494, 496, 486, 534, 504, 504, 502, 512, 497, 540, 533, 456, 547, 496, 527, 550, 520, 487, 508, 550, 512, 523, 553, 508, 496, 479, 509, 516, 485, 499, 488, 535] payload_rows_per_dst=[5620, 5512, 5522, 5546, 5562, 5646, 5604, 5472] dup_rows_per_dst=[2704, 2692, 2684, 2564, 2502, 2646, 2718, 2542] combine_rows_per_src=[5620, 5512, 5522, 5546, 5562, 5646, 5604, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44780 payload_rows_remote=39168 dup_rows=20756 dup_rows_remote=18142 dedup_ratio=0.316711 router_tokens_per_expert=[501, 510, 501, 526, 498, 498, 553, 511, 504, 533, 523, 494, 565, 490, 504, 500, 519, 523, 526, 532, 458, 499, 516, 481, 496, 523, 523, 554, 500, 532, 482, 503, 522, 491, 500, 494, 514, 525, 521, 527, 492, 552, 509, 499, 527, 503, 509, 524, 542, 502, 527, 495, 480, 545, 496, 500, 524, 510, 560, 482, 516, 481, 500, 521] payload_rows_per_dst=[5610, 5612, 5554, 5606, 5544, 5646, 5614, 5594] dup_rows_per_dst=[2586, 2614, 2554, 2620, 2644, 2584, 2560, 2594] combine_rows_per_src=[5610, 5612, 5554, 5606, 5544, 5646, 5614, 5594] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44364 payload_rows_remote=38840 dup_rows=21172 dup_rows_remote=18580 dedup_ratio=0.323059 router_tokens_per_expert=[519, 470, 542, 537, 531, 500, 485, 480, 487, 510, 500, 505, 511, 484, 495, 509, 497, 483, 517, 505, 523, 502, 518, 513, 517, 561, 504, 519, 480, 502, 540, 512, 494, 534, 490, 501, 530, 550, 527, 514, 522, 544, 497, 512, 446, 526, 521, 539, 517, 505, 523, 504, 505, 544, 505, 533, 532, 499, 544, 478, 503, 485, 545, 541] payload_rows_per_dst=[5552, 5488, 5524, 5532, 5560, 5552, 5568, 5588] dup_rows_per_dst=[2576, 2514, 2592, 2738, 2720, 2662, 2704, 2666] combine_rows_per_src=[5552, 5488, 5524, 5532, 5560, 5552, 5568, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281559040 dispatch_meta_bytes=230344 combine_payload_bytes=281559040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281789384 ingress_bytes=391659520 link_bytes=391659520 token_bytes=563348424 weight_movement_bytes=110100480 total_nvlink_bytes=673448904 weight_to_token_ratio=0.195439 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.811603222e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229768 combine_payload_bytes=279293952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279523720 ingress_bytes=389394432 link_bytes=389394432 token_bytes=558817672 weight_movement_bytes=110100480 total_nvlink_bytes=668918152 weight_to_token_ratio=0.197024 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772209616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276756480 dispatch_meta_bytes=229064 combine_payload_bytes=276756480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276985544 ingress_bytes=386856960 link_bytes=386856960 token_bytes=553742024 weight_movement_bytes=110100480 total_nvlink_bytes=663842504 weight_to_token_ratio=0.198830 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.728078805e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=228264 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279622568 ingress_bytes=719796224 link_bytes=719796224 token_bytes=559016872 weight_movement_bytes=440401920 total_nvlink_bytes=999418792 weight_to_token_ratio=0.787815 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251844020e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229296 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279093168 ingress_bytes=388964352 link_bytes=388964352 token_bytes=557957040 weight_movement_bytes=110100480 total_nvlink_bytes=668057520 weight_to_token_ratio=0.197328 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764729818e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279967744 dispatch_meta_bytes=230040 combine_payload_bytes=279967744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280197784 ingress_bytes=390068224 link_bytes=390068224 token_bytes=560165528 weight_movement_bytes=110100480 total_nvlink_bytes=670266008 weight_to_token_ratio=0.196550 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.783927967e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229704 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279337288 ingress_bytes=389208064 link_bytes=389208064 token_bytes=558444872 weight_movement_bytes=110100480 total_nvlink_bytes=668545352 weight_to_token_ratio=0.197156 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768968370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44292 payload_rows_remote=38508 dup_rows=21244 dup_rows_remote=18154 dedup_ratio=0.324158 router_tokens_per_expert=[544, 552, 503, 507, 514, 505, 491, 505, 539, 534, 503, 527, 513, 501, 504, 488, 569, 502, 529, 501, 488, 521, 493, 511, 513, 497, 509, 516, 534, 501, 521, 484, 505, 505, 524, 513, 491, 501, 481, 500, 531, 536, 523, 503, 524, 473, 508, 502, 524, 531, 509, 529, 482, 503, 471, 551, 492, 506, 470, 547, 541, 520, 515, 538] payload_rows_per_dst=[5306, 5614, 5570, 5474, 5512, 5736, 5296, 5784] dup_rows_per_dst=[2370, 2694, 2658, 2676, 2584, 2876, 2296, 3090] combine_rows_per_src=[5306, 5614, 5570, 5474, 5512, 5736, 5296, 5784] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=229176 combine_payload_bytes=279336960 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279566136 ingress_bytes=609638400 link_bytes=609638400 token_bytes=558903096 weight_movement_bytes=330301440 total_nvlink_bytes=889204536 weight_to_token_ratio=0.590982 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060261446e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278390784 dispatch_meta_bytes=229624 combine_payload_bytes=278390784 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278620408 ingress_bytes=498591744 link_bytes=498591744 token_bytes=557011192 weight_movement_bytes=220200960 total_nvlink_bytes=777212152 weight_to_token_ratio=0.395326 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671330471e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588604144 prefetch_bytes=770703360 reduce_bytes=990904320 weight_movement_bytes=1761607680 egress_bytes=2795449584 ingress_bytes=4554762240 link_bytes=4554762240 total_nvlink_bytes=7350211824 weight_to_token_ratio=0.315214 prefetch_share_of_ingress=0.169208 dedup_ratio=0.320526 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=9 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.921480665e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278405120 dispatch_meta_bytes=229680 combine_payload_bytes=278405120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278634800 ingress_bytes=498606080 link_bytes=498606080 token_bytes=557039920 weight_movement_bytes=220200960 total_nvlink_bytes=777240880 weight_to_token_ratio=0.395306 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671579797e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280756224 dispatch_meta_bytes=229240 combine_payload_bytes=280756224 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=280985464 ingress_bytes=390856704 link_bytes=390856704 token_bytes=561741688 weight_movement_bytes=110100480 total_nvlink_bytes=671842168 weight_to_token_ratio=0.195998 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.797640931e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44682 payload_rows_remote=39240 dup_rows=20854 dup_rows_remote=18402 dedup_ratio=0.318207 router_tokens_per_expert=[549, 553, 471, 536, 485, 506, 490, 504, 472, 482, 543, 505, 562, 492, 514, 531, 501, 520, 539, 494, 516, 484, 506, 537, 554, 504, 529, 521, 481, 524, 518, 501, 502, 524, 547, 521, 535, 546, 500, 567, 467, 515, 500, 498, 489, 483, 520, 475, 507, 544, 485, 482, 540, 531, 552, 542, 505, 493, 508, 492, 485, 496, 515, 478] payload_rows_per_dst=[5610, 5616, 5596, 5658, 5674, 5442, 5632, 5454] dup_rows_per_dst=[2578, 2586, 2598, 2606, 2810, 2452, 2734, 2490] combine_rows_per_src=[5610, 5616, 5596, 5658, 5674, 5442, 5632, 5454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44316 payload_rows_remote=38878 dup_rows=21220 dup_rows_remote=18684 dedup_ratio=0.323792 router_tokens_per_expert=[499, 497, 492, 521, 488, 507, 491, 492, 539, 515, 533, 521, 503, 516, 514, 482, 479, 508, 546, 497, 512, 517, 510, 506, 532, 498, 500, 534, 517, 519, 499, 489, 533, 526, 524, 554, 486, 494, 503, 492, 510, 507, 556, 472, 514, 535, 476, 498, 527, 529, 523, 520, 539, 533, 512, 489, 536, 532, 531, 503, 520, 487, 531, 503] payload_rows_per_dst=[5438, 5596, 5574, 5602, 5454, 5562, 5554, 5536] dup_rows_per_dst=[2536, 2650, 2778, 2826, 2518, 2574, 2588, 2750] combine_rows_per_src=[5438, 5596, 5574, 5602, 5454, 5562, 5554, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584673208 prefetch_bytes=880803840 reduce_bytes=1431306240 weight_movement_bytes=2312110080 egress_bytes=2793482680 ingress_bytes=5103300608 link_bytes=5103300608 total_nvlink_bytes=7896783288 weight_to_token_ratio=0.414010 prefetch_share_of_ingress=0.172595 dedup_ratio=0.320795 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=13 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.875479106e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5592763032 prefetch_bytes=990904320 reduce_bytes=660602880 weight_movement_bytes=1651507200 egress_bytes=2797529752 ingress_bytes=4446740480 link_bytes=4446740480 total_nvlink_bytes=7244270232 weight_to_token_ratio=0.295294 prefetch_share_of_ingress=0.222838 dedup_ratio=0.320755 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=6 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.733613058e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=276025344 dispatch_meta_bytes=226648 combine_payload_bytes=276025344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276251992 ingress_bytes=386125824 link_bytes=386125824 token_bytes=552277336 weight_movement_bytes=110100480 total_nvlink_bytes=662377816 weight_to_token_ratio=0.199357 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.715363147e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584131336 prefetch_bytes=1101004800 reduce_bytes=440401920 weight_movement_bytes=1541406720 egress_bytes=2793213192 ingress_bytes=4332324864 link_bytes=4332324864 total_nvlink_bytes=7125538056 weight_to_token_ratio=0.276033 prefetch_share_of_ingress=0.254137 dedup_ratio=0.321448 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=4 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044118 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.534625483e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38838 dup_rows=21130 dup_rows_remote=18568 dedup_ratio=0.322418 router_tokens_per_expert=[520, 521, 539, 522, 545, 519, 502, 498, 492, 523, 484, 519, 506, 505, 505, 547, 541, 508, 496, 535, 543, 511, 505, 554, 518, 529, 514, 511, 460, 512, 513, 511, 513, 480, 495, 467, 554, 515, 494, 547, 515, 526, 521, 511, 520, 476, 492, 515, 539, 487, 490, 525, 487, 538, 580, 455, 496, 483, 515, 531, 511, 495, 477, 510] payload_rows_per_dst=[5608, 5526, 5628, 5492, 5568, 5652, 5494, 5438] dup_rows_per_dst=[2724, 2636, 2758, 2644, 2562, 2500, 2708, 2598] combine_rows_per_src=[5608, 5526, 5628, 5492, 5568, 5652, 5494, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44316 payload_rows_remote=38878 dup_rows=21220 dup_rows_remote=18684 dedup_ratio=0.323792 router_tokens_per_expert=[499, 497, 492, 521, 488, 507, 491, 492, 539, 515, 533, 521, 503, 516, 514, 482, 479, 508, 546, 497, 512, 517, 510, 506, 532, 498, 500, 534, 517, 519, 499, 489, 533, 526, 524, 554, 486, 494, 503, 492, 510, 507, 556, 472, 514, 535, 476, 498, 527, 529, 523, 520, 539, 533, 512, 489, 536, 532, 531, 503, 520, 487, 531, 503] payload_rows_per_dst=[5438, 5596, 5574, 5602, 5454, 5562, 5554, 5536] dup_rows_per_dst=[2536, 2650, 2778, 2826, 2518, 2574, 2588, 2750] combine_rows_per_src=[5438, 5596, 5574, 5602, 5454, 5562, 5554, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38970 dup_rows=21026 dup_rows_remote=18324 dedup_ratio=0.320831 router_tokens_per_expert=[560, 481, 502, 487, 542, 512, 519, 515, 545, 543, 526, 509, 492, 486, 539, 514, 497, 498, 537, 473, 481, 504, 507, 479, 472, 493, 506, 555, 527, 526, 503, 504, 506, 523, 527, 553, 483, 518, 567, 488, 523, 523, 507, 522, 502, 505, 494, 518, 512, 530, 483, 546, 518, 501, 517, 514, 543, 509, 499, 489, 497, 515, 499, 503] payload_rows_per_dst=[5562, 5610, 5462, 5548, 5658, 5566, 5540, 5564] dup_rows_per_dst=[2674, 2698, 2490, 2624, 2672, 2622, 2702, 2544] combine_rows_per_src=[5562, 5610, 5462, 5548, 5658, 5566, 5540, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281272320 dispatch_meta_bytes=230568 combine_payload_bytes=281272320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281502888 ingress_bytes=391372800 link_bytes=391372800 token_bytes=562775208 weight_movement_bytes=110100480 total_nvlink_bytes=672875688 weight_to_token_ratio=0.195638 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.806616690e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38938 dup_rows=21052 dup_rows_remote=18488 dedup_ratio=0.321228 router_tokens_per_expert=[499, 541, 505, 536, 522, 505, 537, 517, 530, 508, 559, 487, 503, 505, 499, 511, 491, 485, 499, 535, 537, 520, 483, 553, 508, 470, 529, 528, 537, 482, 487, 514, 494, 496, 486, 534, 504, 504, 502, 512, 497, 540, 533, 456, 547, 496, 527, 550, 520, 487, 508, 550, 512, 523, 553, 508, 496, 479, 509, 516, 485, 499, 488, 535] payload_rows_per_dst=[5620, 5512, 5522, 5546, 5562, 5646, 5604, 5472] dup_rows_per_dst=[2704, 2692, 2684, 2564, 2502, 2646, 2718, 2542] combine_rows_per_src=[5620, 5512, 5522, 5546, 5562, 5646, 5604, 5472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278677504 dispatch_meta_bytes=230248 combine_payload_bytes=278677504 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278907752 ingress_bytes=719079424 link_bytes=719079424 token_bytes=557585256 weight_movement_bytes=440401920 total_nvlink_bytes=997987176 weight_to_token_ratio=0.789838 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250597387e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5570161808 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=2786225296 ingress_bytes=4985946112 link_bytes=4985946112 total_nvlink_bytes=7772171408 weight_to_token_ratio=0.395322 prefetch_share_of_ingress=0.198739 dedup_ratio=0.321527 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.671380336e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44682 payload_rows_remote=39240 dup_rows=20854 dup_rows_remote=18402 dedup_ratio=0.318207 router_tokens_per_expert=[549, 553, 471, 536, 485, 506, 490, 504, 472, 482, 543, 505, 562, 492, 514, 531, 501, 520, 539, 494, 516, 484, 506, 537, 554, 504, 529, 521, 481, 524, 518, 501, 502, 524, 547, 521, 535, 546, 500, 567, 467, 515, 500, 498, 489, 483, 520, 475, 507, 544, 485, 482, 540, 531, 552, 542, 505, 493, 508, 492, 485, 496, 515, 478] payload_rows_per_dst=[5610, 5616, 5596, 5658, 5674, 5442, 5632, 5454] dup_rows_per_dst=[2578, 2586, 2598, 2606, 2810, 2452, 2734, 2490] combine_rows_per_src=[5610, 5616, 5596, 5658, 5674, 5442, 5632, 5454] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44612 payload_rows_remote=39002 dup_rows=20924 dup_rows_remote=18292 dedup_ratio=0.319275 router_tokens_per_expert=[514, 509, 545, 509, 530, 506, 541, 522, 527, 504, 525, 486, 509, 472, 516, 539, 494, 541, 517, 502, 523, 523, 520, 501, 500, 498, 486, 501, 524, 491, 506, 491, 526, 539, 527, 472, 525, 525, 496, 501, 498, 498, 518, 505, 559, 525, 501, 504, 515, 530, 534, 509, 546, 496, 516, 510, 504, 503, 513, 476, 485, 522, 507, 511] payload_rows_per_dst=[5572, 5604, 5610, 5460, 5648, 5624, 5610, 5484] dup_rows_per_dst=[2780, 2552, 2632, 2534, 2574, 2592, 2702, 2558] combine_rows_per_src=[5572, 5604, 5610, 5460, 5648, 5624, 5610, 5484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5592848576 prefetch_bytes=990904320 reduce_bytes=440401920 weight_movement_bytes=1431306240 egress_bytes=2797572288 ingress_bytes=4226582528 link_bytes=4226582528 total_nvlink_bytes=7024154816 weight_to_token_ratio=0.255917 prefetch_share_of_ingress=0.234446 dedup_ratio=0.320370 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=4 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043750 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.350722170e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5578274656 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=2790281056 ingress_bytes=4990003200 link_bytes=4990003200 total_nvlink_bytes=7780284256 weight_to_token_ratio=0.394747 prefetch_share_of_ingress=0.198578 dedup_ratio=0.320993 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.678436279e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279107584 dispatch_meta_bytes=229704 combine_payload_bytes=279107584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279337288 ingress_bytes=389208064 link_bytes=389208064 token_bytes=558444872 weight_movement_bytes=110100480 total_nvlink_bytes=668545352 weight_to_token_ratio=0.197156 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.768968370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38982 dup_rows=21068 dup_rows_remote=18476 dedup_ratio=0.321472 router_tokens_per_expert=[518, 539, 551, 488, 544, 506, 492, 513, 523, 541, 490, 489, 508, 512, 502, 474, 538, 547, 498, 516, 519, 517, 553, 489, 525, 518, 495, 514, 538, 508, 490, 509, 546, 526, 498, 482, 535, 554, 513, 534, 500, 492, 505, 500, 501, 485, 482, 521, 494, 525, 486, 503, 532, 492, 529, 519, 481, 522, 531, 507, 494, 524, 471, 520] payload_rows_per_dst=[5592, 5486, 5642, 5528, 5698, 5472, 5542, 5508] dup_rows_per_dst=[2710, 2592, 2712, 2666, 2678, 2500, 2618, 2592] combine_rows_per_src=[5592, 5486, 5642, 5528, 5698, 5472, 5542, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278677504 dispatch_meta_bytes=230248 combine_payload_bytes=278677504 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=278907752 ingress_bytes=719079424 link_bytes=719079424 token_bytes=557585256 weight_movement_bytes=440401920 total_nvlink_bytes=997987176 weight_to_token_ratio=0.789838 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250597387e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278390784 dispatch_meta_bytes=229624 combine_payload_bytes=278390784 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278620408 ingress_bytes=498591744 link_bytes=498591744 token_bytes=557011192 weight_movement_bytes=220200960 total_nvlink_bytes=777212152 weight_to_token_ratio=0.395326 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671330471e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=229176 combine_payload_bytes=279336960 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279566136 ingress_bytes=609638400 link_bytes=609638400 token_bytes=558903096 weight_movement_bytes=330301440 total_nvlink_bytes=889204536 weight_to_token_ratio=0.590982 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060261446e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281272320 dispatch_meta_bytes=230568 combine_payload_bytes=281272320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281502888 ingress_bytes=391372800 link_bytes=391372800 token_bytes=562775208 weight_movement_bytes=110100480 total_nvlink_bytes=672875688 weight_to_token_ratio=0.195638 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.806616690e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279566336 dispatch_meta_bytes=229176 combine_payload_bytes=279566336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279795512 ingress_bytes=389666816 link_bytes=389666816 token_bytes=559361848 weight_movement_bytes=110100480 total_nvlink_bytes=669462328 weight_to_token_ratio=0.196832 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776946822e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5592763032 prefetch_bytes=990904320 reduce_bytes=660602880 weight_movement_bytes=1651507200 egress_bytes=2797529752 ingress_bytes=4446740480 link_bytes=4446740480 total_nvlink_bytes=7244270232 weight_to_token_ratio=0.295294 prefetch_share_of_ingress=0.222838 dedup_ratio=0.320755 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=6 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.733613058e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5578274656 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=2790281056 ingress_bytes=4990003200 link_bytes=4990003200 total_nvlink_bytes=7780284256 weight_to_token_ratio=0.394747 prefetch_share_of_ingress=0.198578 dedup_ratio=0.320993 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.678436279e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584131336 prefetch_bytes=1101004800 reduce_bytes=440401920 weight_movement_bytes=1541406720 egress_bytes=2793213192 ingress_bytes=4332324864 link_bytes=4332324864 total_nvlink_bytes=7125538056 weight_to_token_ratio=0.276033 prefetch_share_of_ingress=0.254137 dedup_ratio=0.321448 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=4 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044118 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.534625483e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279422976 dispatch_meta_bytes=229832 combine_payload_bytes=279422976 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279652808 ingress_bytes=499623936 link_bytes=499623936 token_bytes=559075784 weight_movement_bytes=220200960 total_nvlink_bytes=779276744 weight_to_token_ratio=0.393866 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689281987e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584673208 prefetch_bytes=880803840 reduce_bytes=1431306240 weight_movement_bytes=2312110080 egress_bytes=2793482680 ingress_bytes=5103300608 link_bytes=5103300608 total_nvlink_bytes=7896783288 weight_to_token_ratio=0.414010 prefetch_share_of_ingress=0.172595 dedup_ratio=0.320795 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=13 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.875479106e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5592848576 prefetch_bytes=990904320 reduce_bytes=440401920 weight_movement_bytes=1431306240 egress_bytes=2797572288 ingress_bytes=4226582528 link_bytes=4226582528 total_nvlink_bytes=7024154816 weight_to_token_ratio=0.255917 prefetch_share_of_ingress=0.234446 dedup_ratio=0.320370 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=4 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043750 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.350722170e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5586165144 prefetch_bytes=880803840 reduce_bytes=1211105280 weight_movement_bytes=2091909120 egress_bytes=2794229144 ingress_bytes=4883845120 link_bytes=4883845120 total_nvlink_bytes=7678074264 weight_to_token_ratio=0.374480 prefetch_share_of_ingress=0.180350 dedup_ratio=0.320673 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=11 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.493809918e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none - [2026-08-26 16:58:54] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 363.4 | throughput per GPU (TFLOP/s/GPU): 339.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.604866E+00 | loss scale: 1.0 | grad norm: 4.259 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44252 payload_rows_remote=38760 dup_rows=21284 dup_rows_remote=18512 dedup_ratio=0.324768 router_tokens_per_expert=[527, 516, 502, 505, 501, 476, 506, 553, 507, 488, 531, 531, 505, 525, 501, 498, 524, 514, 481, 556, 521, 514, 517, 493, 555, 513, 516, 499, 496, 489, 528, 503, 537, 518, 524, 519, 482, 523, 503, 526, 531, 486, 482, 507, 514, 514, 507, 512, 537, 503, 526, 496, 487, 511, 513, 493, 542, 531, 528, 479, 517, 515, 515, 499] payload_rows_per_dst=[5508, 5582, 5514, 5566, 5492, 5494, 5468, 5628] dup_rows_per_dst=[2664, 2590, 2726, 2632, 2772, 2612, 2664, 2624] combine_rows_per_src=[5508, 5582, 5514, 5566, 5492, 5494, 5468, 5628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44608 payload_rows_remote=39072 dup_rows=20928 dup_rows_remote=18402 dedup_ratio=0.319336 router_tokens_per_expert=[540, 511, 510, 507, 517, 541, 501, 525, 521, 540, 525, 503, 486, 454, 520, 482, 536, 538, 483, 522, 483, 537, 551, 500, 499, 501, 502, 509, 500, 532, 509, 502, 514, 474, 474, 524, 548, 502, 542, 529, 516, 487, 474, 533, 521, 520, 526, 485, 506, 484, 551, 504, 523, 518, 502, 545, 491, 512, 503, 505, 535, 514, 510, 509] payload_rows_per_dst=[5626, 5536, 5590, 5550, 5592, 5574, 5596, 5544] dup_rows_per_dst=[2678, 2526, 2710, 2558, 2622, 2550, 2670, 2614] combine_rows_per_src=[5626, 5536, 5590, 5550, 5592, 5574, 5596, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44740 payload_rows_remote=39280 dup_rows=20796 dup_rows_remote=18306 dedup_ratio=0.317322 router_tokens_per_expert=[554, 527, 484, 524, 575, 522, 528, 513, 501, 511, 496, 477, 515, 508, 465, 502, 488, 488, 474, 474, 529, 523, 481, 531, 547, 512, 505, 517, 518, 491, 525, 526, 542, 553, 466, 519, 520, 502, 548, 469, 504, 524, 479, 479, 497, 514, 528, 498, 526, 508, 509, 541, 521, 535, 509, 530, 518, 524, 501, 501, 529, 502, 521, 520] payload_rows_per_dst=[5762, 5460, 5518, 5602, 5646, 5512, 5654, 5586] dup_rows_per_dst=[2692, 2490, 2458, 2680, 2592, 2534, 2704, 2646] combine_rows_per_src=[5762, 5460, 5518, 5602, 5646, 5512, 5654, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281559040 dispatch_meta_bytes=230344 combine_payload_bytes=281559040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281789384 ingress_bytes=391659520 link_bytes=391659520 token_bytes=563348424 weight_movement_bytes=110100480 total_nvlink_bytes=673448904 weight_to_token_ratio=0.195439 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.811603222e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588604144 prefetch_bytes=770703360 reduce_bytes=990904320 weight_movement_bytes=1761607680 egress_bytes=2795449584 ingress_bytes=4554762240 link_bytes=4554762240 total_nvlink_bytes=7350211824 weight_to_token_ratio=0.315214 prefetch_share_of_ingress=0.169208 dedup_ratio=0.320526 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=9 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.921480665e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none + [2026-08-26 18:10:59] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 368.7 | throughput per GPU (TFLOP/s/GPU): 334.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.605629E+00 | loss scale: 1.0 | grad norm: 4.240 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=38968 dup_rows=21036 dup_rows_remote=18460 dedup_ratio=0.320984 router_tokens_per_expert=[491, 475, 553, 543, 509, 502, 502, 530, 528, 552, 514, 470, 503, 536, 499, 505, 530, 513, 518, 511, 514, 517, 533, 511, 530, 518, 503, 501, 510, 519, 491, 521, 493, 478, 527, 503, 516, 528, 516, 526, 539, 467, 530, 457, 505, 496, 527, 533, 509, 507, 526, 496, 510, 479, 490, 537, 510, 474, 506, 533, 553, 501, 524, 520] payload_rows_per_dst=[5518, 5596, 5606, 5592, 5576, 5532, 5512, 5568] dup_rows_per_dst=[2692, 2618, 2688, 2594, 2598, 2576, 2596, 2674] combine_rows_per_src=[5518, 5596, 5606, 5592, 5576, 5532, 5512, 5568] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44508 payload_rows_remote=38792 dup_rows=21028 dup_rows_remote=18262 dedup_ratio=0.320862 router_tokens_per_expert=[520, 498, 518, 527, 512, 488, 532, 489, 492, 509, 526, 549, 525, 498, 495, 523, 515, 522, 558, 481, 492, 554, 479, 529, 528, 457, 509, 529, 544, 517, 537, 556, 492, 494, 540, 499, 508, 496, 518, 498, 528, 523, 472, 524, 518, 457, 503, 482, 504, 527, 495, 512, 522, 460, 536, 516, 528, 488, 515, 502, 544, 529, 505, 525] payload_rows_per_dst=[5588, 5606, 5522, 5510, 5554, 5454, 5558, 5716] dup_rows_per_dst=[2580, 2628, 2506, 2628, 2768, 2560, 2592, 2766] combine_rows_per_src=[5588, 5606, 5522, 5510, 5554, 5454, 5558, 5716] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44374 payload_rows_remote=38802 dup_rows=21162 dup_rows_remote=18484 dedup_ratio=0.322906 router_tokens_per_expert=[475, 504, 512, 515, 514, 511, 538, 506, 512, 499, 529, 524, 484, 484, 496, 551, 499, 512, 559, 504, 515, 523, 579, 507, 501, 553, 512, 479, 534, 502, 514, 519, 525, 463, 535, 493, 520, 488, 518, 478, 493, 459, 492, 524, 516, 478, 494, 519, 501, 518, 525, 532, 526, 484, 514, 525, 530, 522, 521, 513, 534, 525, 509, 528] payload_rows_per_dst=[5560, 5558, 5568, 5508, 5484, 5406, 5572, 5718] dup_rows_per_dst=[2590, 2600, 2828, 2720, 2556, 2544, 2678, 2646] combine_rows_per_src=[5560, 5558, 5568, 5508, 5484, 5406, 5572, 5718] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38988 dup_rows=20976 dup_rows_remote=18362 dedup_ratio=0.320068 router_tokens_per_expert=[471, 543, 498, 508, 508, 522, 535, 492, 498, 520, 502, 480, 502, 542, 517, 517, 523, 507, 553, 511, 531, 485, 496, 487, 508, 490, 525, 532, 512, 536, 490, 525, 493, 520, 521, 512, 523, 514, 503, 494, 484, 545, 513, 494, 496, 505, 516, 506, 503, 525, 536, 520, 515, 582, 481, 488, 490, 536, 491, 534, 540, 514, 486, 522] payload_rows_per_dst=[5462, 5548, 5572, 5628, 5568, 5570, 5620, 5592] dup_rows_per_dst=[2692, 2608, 2614, 2608, 2592, 2548, 2680, 2634] combine_rows_per_src=[5462, 5548, 5572, 5628, 5568, 5570, 5620, 5592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=38968 dup_rows=21036 dup_rows_remote=18460 dedup_ratio=0.320984 router_tokens_per_expert=[491, 475, 553, 543, 509, 502, 502, 530, 528, 552, 514, 470, 503, 536, 499, 505, 530, 513, 518, 511, 514, 517, 533, 511, 530, 518, 503, 501, 510, 519, 491, 521, 493, 478, 527, 503, 516, 528, 516, 526, 539, 467, 530, 457, 505, 496, 527, 533, 509, 507, 526, 496, 510, 479, 490, 537, 510, 474, 506, 533, 553, 501, 524, 520] payload_rows_per_dst=[5518, 5596, 5606, 5592, 5576, 5532, 5512, 5568] dup_rows_per_dst=[2692, 2618, 2688, 2594, 2598, 2576, 2596, 2674] combine_rows_per_src=[5518, 5596, 5606, 5592, 5576, 5532, 5512, 5568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44612 payload_rows_remote=38964 dup_rows=20924 dup_rows_remote=18306 dedup_ratio=0.319275 router_tokens_per_expert=[540, 520, 506, 538, 494, 500, 533, 500, 499, 534, 535, 511, 481, 501, 483, 506, 500, 529, 474, 490, 496, 493, 538, 526, 510, 513, 534, 497, 522, 533, 499, 525, 515, 536, 511, 521, 494, 512, 515, 509, 520, 501, 507, 536, 479, 503, 543, 493, 517, 525, 513, 501, 516, 474, 509, 532, 489, 529, 515, 521, 527, 519, 503, 523] payload_rows_per_dst=[5626, 5526, 5526, 5648, 5642, 5580, 5502, 5562] dup_rows_per_dst=[2636, 2574, 2566, 2618, 2584, 2584, 2672, 2690] combine_rows_per_src=[5626, 5526, 5526, 5648, 5642, 5580, 5502, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44252 payload_rows_remote=38760 dup_rows=21284 dup_rows_remote=18512 dedup_ratio=0.324768 router_tokens_per_expert=[527, 516, 502, 505, 501, 476, 506, 553, 507, 488, 531, 531, 505, 525, 501, 498, 524, 514, 481, 556, 521, 514, 517, 493, 555, 513, 516, 499, 496, 489, 528, 503, 537, 518, 524, 519, 482, 523, 503, 526, 531, 486, 482, 507, 514, 514, 507, 512, 537, 503, 526, 496, 487, 511, 513, 493, 542, 531, 528, 479, 517, 515, 515, 499] payload_rows_per_dst=[5508, 5582, 5514, 5566, 5492, 5494, 5468, 5628] dup_rows_per_dst=[2664, 2590, 2726, 2632, 2772, 2612, 2664, 2624] combine_rows_per_src=[5508, 5582, 5514, 5566, 5492, 5494, 5468, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=38988 dup_rows=20976 dup_rows_remote=18362 dedup_ratio=0.320068 router_tokens_per_expert=[471, 543, 498, 508, 508, 522, 535, 492, 498, 520, 502, 480, 502, 542, 517, 517, 523, 507, 553, 511, 531, 485, 496, 487, 508, 490, 525, 532, 512, 536, 490, 525, 493, 520, 521, 512, 523, 514, 503, 494, 484, 545, 513, 494, 496, 505, 516, 506, 503, 525, 536, 520, 515, 582, 481, 488, 490, 536, 491, 534, 540, 514, 486, 522] payload_rows_per_dst=[5462, 5548, 5572, 5628, 5568, 5570, 5620, 5592] dup_rows_per_dst=[2692, 2608, 2614, 2608, 2592, 2548, 2680, 2634] combine_rows_per_src=[5462, 5548, 5572, 5628, 5568, 5570, 5620, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=229088 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278060768 ingress_bytes=387932160 link_bytes=387932160 token_bytes=555892448 weight_movement_bytes=110100480 total_nvlink_bytes=665992928 weight_to_token_ratio=0.198061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746778301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280068096 dispatch_meta_bytes=229896 combine_payload_bytes=280068096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280297992 ingress_bytes=390168576 link_bytes=390168576 token_bytes=560366088 weight_movement_bytes=110100480 total_nvlink_bytes=670466568 weight_to_token_ratio=0.196480 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785673254e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38944 dup_rows=21092 dup_rows_remote=18476 dedup_ratio=0.321838 router_tokens_per_expert=[505, 497, 515, 515, 557, 488, 463, 480, 519, 509, 515, 500, 530, 519, 486, 554, 481, 528, 480, 509, 502, 518, 528, 524, 513, 530, 517, 549, 436, 512, 518, 513, 525, 503, 498, 497, 538, 481, 537, 489, 503, 533, 510, 479, 550, 476, 585, 504, 512, 488, 521, 509, 489, 547, 543, 525, 527, 507, 494, 502, 502, 532, 509, 543] payload_rows_per_dst=[5500, 5676, 5534, 5506, 5488, 5914, 5540, 5286] dup_rows_per_dst=[2616, 2764, 2466, 2670, 2648, 3038, 2616, 2274] combine_rows_per_src=[5500, 5676, 5534, 5506, 5488, 5914, 5540, 5286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229712 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279552336 ingress_bytes=389423104 link_bytes=389423104 token_bytes=558874960 weight_movement_bytes=110100480 total_nvlink_bytes=668975440 weight_to_token_ratio=0.197004 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772708270e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278061056 dispatch_meta_bytes=228216 combine_payload_bytes=278061056 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278289272 ingress_bytes=498262016 link_bytes=498262016 token_bytes=556350328 weight_movement_bytes=220200960 total_nvlink_bytes=776551288 weight_to_token_ratio=0.395796 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.665595958e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278132736 dispatch_meta_bytes=229144 combine_payload_bytes=278132736 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278361880 ingress_bytes=608434176 link_bytes=608434176 token_bytes=556494616 weight_movement_bytes=330301440 total_nvlink_bytes=886796056 weight_to_token_ratio=0.593539 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058167102e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44608 payload_rows_remote=39072 dup_rows=20928 dup_rows_remote=18402 dedup_ratio=0.319336 router_tokens_per_expert=[540, 511, 510, 507, 517, 541, 501, 525, 521, 540, 525, 503, 486, 454, 520, 482, 536, 538, 483, 522, 483, 537, 551, 500, 499, 501, 502, 509, 500, 532, 509, 502, 514, 474, 474, 524, 548, 502, 542, 529, 516, 487, 474, 533, 521, 520, 526, 485, 506, 484, 551, 504, 523, 518, 502, 545, 491, 512, 503, 505, 535, 514, 510, 509] payload_rows_per_dst=[5626, 5536, 5590, 5550, 5592, 5574, 5596, 5544] dup_rows_per_dst=[2678, 2526, 2710, 2558, 2622, 2550, 2670, 2614] combine_rows_per_src=[5626, 5536, 5590, 5550, 5592, 5574, 5596, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229080 combine_payload_bytes=279293952 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279523032 ingress_bytes=499494912 link_bytes=499494912 token_bytes=558816984 weight_movement_bytes=220200960 total_nvlink_bytes=779017944 weight_to_token_ratio=0.394048 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687038048e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=229088 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278060768 ingress_bytes=387932160 link_bytes=387932160 token_bytes=555892448 weight_movement_bytes=110100480 total_nvlink_bytes=665992928 weight_to_token_ratio=0.198061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746778301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=229400 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279695384 ingress_bytes=609767424 link_bytes=609767424 token_bytes=559161368 weight_movement_bytes=330301440 total_nvlink_bytes=889462808 weight_to_token_ratio=0.590709 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060485840e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229712 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279552336 ingress_bytes=389423104 link_bytes=389423104 token_bytes=558874960 weight_movement_bytes=110100480 total_nvlink_bytes=668975440 weight_to_token_ratio=0.197004 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.772708270e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279293952 dispatch_meta_bytes=229080 combine_payload_bytes=279293952 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279523032 ingress_bytes=499494912 link_bytes=499494912 token_bytes=558816984 weight_movement_bytes=220200960 total_nvlink_bytes=779017944 weight_to_token_ratio=0.394048 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687038048e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=229680 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279380272 ingress_bytes=389251072 link_bytes=389251072 token_bytes=558530864 weight_movement_bytes=110100480 total_nvlink_bytes=668631344 weight_to_token_ratio=0.197125 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769716350e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38826 dup_rows=21116 dup_rows_remote=18494 dedup_ratio=0.322205 router_tokens_per_expert=[497, 474, 529, 483, 541, 543, 494, 522, 505, 504, 521, 500, 463, 503, 471, 566, 515, 522, 491, 527, 500, 511, 485, 502, 513, 531, 511, 541, 506, 512, 560, 504, 502, 558, 543, 509, 488, 517, 477, 514, 513, 521, 501, 536, 493, 506, 474, 506, 542, 510, 511, 521, 530, 504, 542, 522, 513, 508, 505, 523, 504, 543, 478, 507] payload_rows_per_dst=[5546, 5530, 5500, 5640, 5594, 5464, 5578, 5568] dup_rows_per_dst=[2620, 2536, 2606, 2716, 2622, 2636, 2786, 2594] combine_rows_per_src=[5546, 5530, 5500, 5640, 5594, 5464, 5578, 5568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38894 dup_rows=20992 dup_rows_remote=18264 dedup_ratio=0.320312 router_tokens_per_expert=[489, 529, 466, 519, 515, 509, 524, 510, 524, 469, 537, 569, 528, 559, 484, 519, 482, 533, 509, 510, 505, 489, 464, 549, 501, 536, 521, 546, 507, 492, 524, 527, 490, 503, 481, 534, 508, 506, 533, 532, 518, 503, 517, 517, 510, 507, 490, 511, 470, 491, 465, 495, 516, 500, 538, 533, 543, 487, 537, 522, 498, 522, 551, 495] payload_rows_per_dst=[5498, 5650, 5490, 5624, 5578, 5570, 5544, 5590] dup_rows_per_dst=[2624, 2728, 2592, 2684, 2596, 2576, 2472, 2720] combine_rows_per_src=[5498, 5650, 5490, 5624, 5578, 5570, 5544, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280068096 dispatch_meta_bytes=229896 combine_payload_bytes=280068096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280297992 ingress_bytes=390168576 link_bytes=390168576 token_bytes=560366088 weight_movement_bytes=110100480 total_nvlink_bytes=670466568 weight_to_token_ratio=0.196480 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785673254e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38924 dup_rows=21052 dup_rows_remote=18412 dedup_ratio=0.321228 router_tokens_per_expert=[492, 496, 519, 528, 487, 520, 525, 499, 479, 487, 503, 533, 523, 487, 540, 521, 506, 508, 508, 520, 553, 517, 509, 535, 572, 508, 502, 530, 482, 520, 478, 518, 503, 524, 479, 504, 511, 516, 470, 526, 521, 535, 489, 542, 501, 496, 510, 506, 506, 516, 504, 503, 525, 489, 483, 533, 555, 499, 522, 521, 551, 503, 491, 529] payload_rows_per_dst=[5540, 5544, 5594, 5562, 5506, 5560, 5502, 5676] dup_rows_per_dst=[2592, 2602, 2718, 2658, 2560, 2640, 2616, 2666] combine_rows_per_src=[5540, 5544, 5594, 5562, 5506, 5560, 5502, 5676] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44560 payload_rows_remote=39066 dup_rows=20976 dup_rows_remote=18404 dedup_ratio=0.320068 router_tokens_per_expert=[534, 509, 511, 516, 500, 476, 495, 514, 568, 510, 556, 538, 546, 491, 509, 504, 543, 488, 506, 487, 518, 543, 488, 493, 519, 514, 546, 520, 517, 521, 515, 501, 511, 491, 489, 496, 517, 489, 512, 526, 525, 496, 516, 527, 474, 510, 494, 519, 540, 531, 538, 500, 486, 558, 491, 504, 497, 503, 512, 511, 520, 522, 460, 507] payload_rows_per_dst=[5568, 5682, 5472, 5548, 5566, 5630, 5600, 5494] dup_rows_per_dst=[2542, 2762, 2464, 2710, 2588, 2642, 2696, 2572] combine_rows_per_src=[5568, 5682, 5472, 5548, 5566, 5630, 5600, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38956 dup_rows=21122 dup_rows_remote=18570 dedup_ratio=0.322296 router_tokens_per_expert=[463, 479, 518, 492, 531, 513, 525, 538, 509, 482, 494, 540, 508, 514, 472, 505, 488, 515, 551, 508, 501, 513, 502, 539, 486, 507, 516, 477, 495, 508, 509, 507, 509, 504, 552, 518, 551, 498, 517, 522, 516, 490, 497, 518, 502, 524, 526, 517, 483, 490, 532, 501, 534, 517, 526, 553, 495, 495, 525, 494, 538, 577, 493, 549] payload_rows_per_dst=[5524, 5464, 5636, 5458, 5638, 5534, 5578, 5582] dup_rows_per_dst=[2594, 2584, 2598, 2552, 2704, 2646, 2694, 2750] combine_rows_per_src=[5524, 5464, 5636, 5458, 5638, 5534, 5578, 5582] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44820 payload_rows_remote=39164 dup_rows=20716 dup_rows_remote=18074 dedup_ratio=0.316101 router_tokens_per_expert=[514, 477, 510, 531, 503, 500, 487, 496, 460, 514, 520, 558, 512, 549, 508, 475, 487, 514, 495, 508, 586, 489, 494, 573, 532, 507, 517, 522, 490, 515, 527, 515, 509, 517, 510, 519, 506, 507, 475, 518, 510, 507, 517, 497, 521, 513, 514, 527, 522, 515, 523, 485, 528, 532, 518, 526, 528, 526, 536, 525, 461, 473, 486, 532] payload_rows_per_dst=[5488, 5604, 5628, 5678, 5598, 5596, 5656, 5572] dup_rows_per_dst=[2548, 2588, 2664, 2572, 2524, 2616, 2642, 2562] combine_rows_per_src=[5488, 5604, 5628, 5678, 5598, 5596, 5656, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38780 dup_rows=21058 dup_rows_remote=18366 dedup_ratio=0.321320 router_tokens_per_expert=[474, 497, 513, 524, 528, 504, 523, 546, 487, 503, 481, 560, 508, 562, 505, 542, 547, 520, 484, 523, 545, 545, 507, 524, 500, 506, 520, 501, 507, 500, 559, 503, 538, 489, 517, 510, 507, 492, 504, 544, 522, 528, 500, 506, 502, 519, 514, 505, 520, 493, 524, 474, 468, 484, 529, 522, 516, 546, 468, 498, 491, 496, 486, 508] payload_rows_per_dst=[5558, 5532, 5698, 5542, 5570, 5656, 5500, 5422] dup_rows_per_dst=[2660, 2764, 2692, 2650, 2632, 2536, 2528, 2596] combine_rows_per_src=[5558, 5532, 5698, 5542, 5570, 5656, 5500, 5422] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229280 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278534048 ingress_bytes=388405248 link_bytes=388405248 token_bytes=556838816 weight_movement_bytes=110100480 total_nvlink_bytes=666939296 weight_to_token_ratio=0.197724 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755006080e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38826 dup_rows=21116 dup_rows_remote=18494 dedup_ratio=0.322205 router_tokens_per_expert=[497, 474, 529, 483, 541, 543, 494, 522, 505, 504, 521, 500, 463, 503, 471, 566, 515, 522, 491, 527, 500, 511, 485, 502, 513, 531, 511, 541, 506, 512, 560, 504, 502, 558, 543, 509, 488, 517, 477, 514, 513, 521, 501, 536, 493, 506, 474, 506, 542, 510, 511, 521, 530, 504, 542, 522, 513, 508, 505, 523, 504, 543, 478, 507] payload_rows_per_dst=[5546, 5530, 5500, 5640, 5594, 5464, 5578, 5568] dup_rows_per_dst=[2620, 2536, 2606, 2716, 2622, 2636, 2786, 2594] combine_rows_per_src=[5546, 5530, 5500, 5640, 5594, 5464, 5578, 5568] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38780 dup_rows=21058 dup_rows_remote=18366 dedup_ratio=0.321320 router_tokens_per_expert=[474, 497, 513, 524, 528, 504, 523, 546, 487, 503, 481, 560, 508, 562, 505, 542, 547, 520, 484, 523, 545, 545, 507, 524, 500, 506, 520, 501, 507, 500, 559, 503, 538, 489, 517, 510, 507, 492, 504, 544, 522, 528, 500, 506, 502, 519, 514, 505, 520, 493, 524, 474, 468, 484, 529, 522, 516, 546, 468, 498, 491, 496, 486, 508] payload_rows_per_dst=[5558, 5532, 5698, 5542, 5570, 5656, 5500, 5422] dup_rows_per_dst=[2660, 2764, 2692, 2650, 2632, 2536, 2528, 2596] combine_rows_per_src=[5558, 5532, 5698, 5542, 5570, 5656, 5500, 5422] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=228632 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279020824 ingress_bytes=498993152 link_bytes=498993152 token_bytes=557813016 weight_movement_bytes=220200960 total_nvlink_bytes=778013976 weight_to_token_ratio=0.394758 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678311616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38924 dup_rows=21052 dup_rows_remote=18412 dedup_ratio=0.321228 router_tokens_per_expert=[492, 496, 519, 528, 487, 520, 525, 499, 479, 487, 503, 533, 523, 487, 540, 521, 506, 508, 508, 520, 553, 517, 509, 535, 572, 508, 502, 530, 482, 520, 478, 518, 503, 524, 479, 504, 511, 516, 470, 526, 521, 535, 489, 542, 501, 496, 510, 506, 506, 516, 504, 503, 525, 489, 483, 533, 555, 499, 522, 521, 551, 503, 491, 529] payload_rows_per_dst=[5540, 5544, 5594, 5562, 5506, 5560, 5502, 5676] dup_rows_per_dst=[2592, 2602, 2718, 2658, 2560, 2640, 2616, 2666] combine_rows_per_src=[5540, 5544, 5594, 5562, 5506, 5560, 5502, 5676] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38956 dup_rows=21122 dup_rows_remote=18570 dedup_ratio=0.322296 router_tokens_per_expert=[463, 479, 518, 492, 531, 513, 525, 538, 509, 482, 494, 540, 508, 514, 472, 505, 488, 515, 551, 508, 501, 513, 502, 539, 486, 507, 516, 477, 495, 508, 509, 507, 509, 504, 552, 518, 551, 498, 517, 522, 516, 490, 497, 518, 502, 524, 526, 517, 483, 490, 532, 501, 534, 517, 526, 553, 495, 495, 525, 494, 538, 577, 493, 549] payload_rows_per_dst=[5524, 5464, 5636, 5458, 5638, 5534, 5578, 5582] dup_rows_per_dst=[2594, 2584, 2598, 2552, 2704, 2646, 2694, 2750] combine_rows_per_src=[5524, 5464, 5636, 5458, 5638, 5534, 5578, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229344 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279236576 ingress_bytes=389107712 link_bytes=389107712 token_bytes=558243808 weight_movement_bytes=110100480 total_nvlink_bytes=668344288 weight_to_token_ratio=0.197227 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767223084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280025088 dispatch_meta_bytes=229880 combine_payload_bytes=280025088 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280254968 ingress_bytes=500226048 link_bytes=500226048 token_bytes=560280056 weight_movement_bytes=220200960 total_nvlink_bytes=780481016 weight_to_token_ratio=0.393019 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.699753705e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280727552 dispatch_meta_bytes=228952 combine_payload_bytes=280727552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280956504 ingress_bytes=390828032 link_bytes=390828032 token_bytes=561684056 weight_movement_bytes=110100480 total_nvlink_bytes=671784536 weight_to_token_ratio=0.196019 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.797142278e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44650 payload_rows_remote=38900 dup_rows=20886 dup_rows_remote=18002 dedup_ratio=0.318695 router_tokens_per_expert=[528, 538, 517, 505, 529, 524, 505, 482, 531, 553, 540, 503, 491, 496, 470, 510, 534, 539, 469, 518, 529, 503, 511, 489, 498, 495, 512, 508, 486, 523, 483, 544, 539, 526, 536, 513, 522, 495, 531, 468, 534, 473, 507, 497, 534, 524, 501, 547, 532, 473, 557, 476, 494, 537, 507, 492, 512, 534, 513, 525, 495, 500, 516, 495] payload_rows_per_dst=[5750, 5474, 5600, 5480, 5608, 5634, 5512, 5592] dup_rows_per_dst=[2884, 2492, 2584, 2462, 2652, 2600, 2632, 2580] combine_rows_per_src=[5750, 5474, 5600, 5480, 5608, 5634, 5512, 5592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=228584 combine_payload_bytes=277975040 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278203624 ingress_bytes=608276480 link_bytes=608276480 token_bytes=556178664 weight_movement_bytes=330301440 total_nvlink_bytes=886480104 weight_to_token_ratio=0.593877 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057892843e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229344 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279236576 ingress_bytes=389107712 link_bytes=389107712 token_bytes=558243808 weight_movement_bytes=110100480 total_nvlink_bytes=668344288 weight_to_token_ratio=0.197227 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767223084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=230104 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279466712 ingress_bytes=609538048 link_bytes=609538048 token_bytes=558703320 weight_movement_bytes=330301440 total_nvlink_bytes=889004760 weight_to_token_ratio=0.591193 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060086917e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280727552 dispatch_meta_bytes=228952 combine_payload_bytes=280727552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280956504 ingress_bytes=390828032 link_bytes=390828032 token_bytes=561684056 weight_movement_bytes=110100480 total_nvlink_bytes=671784536 weight_to_token_ratio=0.196019 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.797142278e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278304768 dispatch_meta_bytes=229280 combine_payload_bytes=278304768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278534048 ingress_bytes=388405248 link_bytes=388405248 token_bytes=556838816 weight_movement_bytes=110100480 total_nvlink_bytes=666939296 weight_to_token_ratio=0.197724 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755006080e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38894 dup_rows=20992 dup_rows_remote=18264 dedup_ratio=0.320312 router_tokens_per_expert=[489, 529, 466, 519, 515, 509, 524, 510, 524, 469, 537, 569, 528, 559, 484, 519, 482, 533, 509, 510, 505, 489, 464, 549, 501, 536, 521, 546, 507, 492, 524, 527, 490, 503, 481, 534, 508, 506, 533, 532, 518, 503, 517, 517, 510, 507, 490, 511, 470, 491, 465, 495, 516, 500, 538, 533, 543, 487, 537, 522, 498, 522, 551, 495] payload_rows_per_dst=[5498, 5650, 5490, 5624, 5578, 5570, 5544, 5590] dup_rows_per_dst=[2624, 2728, 2592, 2684, 2596, 2576, 2472, 2720] combine_rows_per_src=[5498, 5650, 5490, 5624, 5578, 5570, 5544, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277975040 dispatch_meta_bytes=228584 combine_payload_bytes=277975040 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278203624 ingress_bytes=608276480 link_bytes=608276480 token_bytes=556178664 weight_movement_bytes=330301440 total_nvlink_bytes=886480104 weight_to_token_ratio=0.593877 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057892843e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=227608 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279062808 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557898008 weight_movement_bytes=110100480 total_nvlink_bytes=667998488 weight_to_token_ratio=0.197349 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38912 dup_rows=21068 dup_rows_remote=18472 dedup_ratio=0.321472 router_tokens_per_expert=[494, 500, 495, 501, 535, 509, 516, 512, 525, 495, 511, 525, 495, 494, 527, 461, 530, 519, 547, 511, 524, 511, 513, 546, 507, 512, 492, 481, 518, 505, 554, 534, 511, 497, 520, 469, 512, 506, 513, 548, 517, 508, 511, 523, 539, 493, 536, 513, 538, 514, 500, 544, 502, 520, 495, 471, 558, 453, 506, 534, 527, 500, 486, 505] payload_rows_per_dst=[5542, 5508, 5630, 5620, 5556, 5486, 5602, 5524] dup_rows_per_dst=[2582, 2558, 2772, 2586, 2596, 2794, 2566, 2614] combine_rows_per_src=[5542, 5508, 5630, 5620, 5556, 5486, 5602, 5524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38942 dup_rows=20984 dup_rows_remote=18218 dedup_ratio=0.320190 router_tokens_per_expert=[534, 489, 501, 521, 497, 505, 498, 533, 506, 484, 533, 551, 552, 537, 519, 506, 507, 510, 496, 478, 537, 545, 541, 515, 508, 530, 551, 547, 487, 537, 498, 498, 538, 505, 512, 476, 500, 516, 496, 520, 465, 504, 504, 531, 514, 494, 541, 508, 508, 495, 492, 529, 534, 460, 500, 515, 536, 499, 517, 470, 509, 527, 480, 522] payload_rows_per_dst=[5588, 5610, 5636, 5584, 5548, 5486, 5542, 5558] dup_rows_per_dst=[2568, 2766, 2622, 2728, 2578, 2636, 2524, 2562] combine_rows_per_src=[5588, 5610, 5636, 5584, 5548, 5486, 5542, 5558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=228632 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279020824 ingress_bytes=498993152 link_bytes=498993152 token_bytes=557813016 weight_movement_bytes=220200960 total_nvlink_bytes=778013976 weight_to_token_ratio=0.394758 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678311616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=39102 dup_rows=20938 dup_rows_remote=18366 dedup_ratio=0.319489 router_tokens_per_expert=[504, 543, 509, 512, 530, 543, 508, 492, 533, 525, 535, 514, 523, 498, 512, 524, 481, 523, 506, 499, 530, 518, 515, 518, 478, 531, 521, 515, 525, 520, 518, 530, 530, 499, 491, 513, 481, 541, 531, 539, 502, 515, 494, 514, 489, 497, 501, 522, 511, 468, 519, 487, 507, 524, 527, 502, 509, 507, 509, 502, 507, 516, 497, 484] payload_rows_per_dst=[5592, 5684, 5542, 5612, 5648, 5496, 5550, 5474] dup_rows_per_dst=[2690, 2644, 2638, 2664, 2602, 2572, 2540, 2588] combine_rows_per_src=[5592, 5684, 5542, 5612, 5648, 5496, 5550, 5474] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44308 payload_rows_remote=38682 dup_rows=21228 dup_rows_remote=18418 dedup_ratio=0.323914 router_tokens_per_expert=[514, 514, 537, 515, 520, 499, 482, 523, 554, 531, 479, 488, 487, 504, 542, 520, 536, 505, 507, 512, 550, 493, 512, 517, 506, 493, 479, 509, 525, 492, 526, 502, 516, 508, 523, 460, 529, 568, 561, 495, 526, 508, 509, 488, 533, 497, 521, 475, 483, 519, 512, 518, 522, 522, 495, 519, 479, 506, 541, 539, 499, 504, 491, 529] payload_rows_per_dst=[5538, 5492, 5462, 5496, 5546, 5554, 5594, 5626] dup_rows_per_dst=[2670, 2538, 2542, 2566, 2660, 2676, 2766, 2810] combine_rows_per_src=[5538, 5492, 5462, 5496, 5546, 5554, 5594, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38880 dup_rows=21088 dup_rows_remote=18446 dedup_ratio=0.321777 router_tokens_per_expert=[503, 497, 514, 528, 507, 505, 476, 501, 519, 509, 527, 493, 471, 490, 537, 548, 539, 480, 531, 520, 524, 489, 537, 507, 509, 514, 510, 523, 548, 491, 499, 511, 546, 490, 543, 489, 504, 501, 503, 503, 496, 526, 507, 508, 533, 496, 506, 519, 499, 528, 520, 516, 511, 517, 507, 530, 495, 491, 484, 520, 544, 528, 520, 531] payload_rows_per_dst=[5514, 5554, 5604, 5568, 5524, 5636, 5474, 5574] dup_rows_per_dst=[2548, 2634, 2650, 2642, 2634, 2546, 2782, 2652] combine_rows_per_src=[5514, 5554, 5604, 5568, 5524, 5636, 5474, 5574] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44586 payload_rows_remote=39078 dup_rows=20950 dup_rows_remote=18410 dedup_ratio=0.319672 router_tokens_per_expert=[544, 466, 502, 530, 502, 491, 517, 521, 508, 560, 530, 494, 551, 536, 496, 531, 490, 505, 489, 522, 496, 503, 534, 531, 492, 526, 501, 539, 504, 526, 536, 512, 511, 513, 483, 488, 563, 497, 489, 513, 491, 512, 550, 519, 509, 510, 542, 523, 528, 480, 494, 496, 508, 505, 516, 497, 493, 482, 514, 542, 475, 518, 532, 490] payload_rows_per_dst=[5550, 5664, 5562, 5600, 5552, 5614, 5508, 5536] dup_rows_per_dst=[2596, 2748, 2578, 2672, 2562, 2698, 2540, 2556] combine_rows_per_src=[5550, 5664, 5562, 5600, 5552, 5614, 5508, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44440 payload_rows_remote=38984 dup_rows=21096 dup_rows_remote=18608 dedup_ratio=0.321899 router_tokens_per_expert=[510, 503, 529, 483, 520, 522, 492, 483, 514, 519, 546, 472, 518, 523, 491, 497, 481, 519, 467, 477, 485, 503, 521, 519, 498, 493, 476, 507, 547, 472, 525, 525, 541, 492, 551, 511, 524, 536, 520, 528, 535, 514, 503, 493, 547, 532, 491, 515, 531, 523, 481, 503, 500, 497, 531, 568, 538, 512, 543, 494, 514, 503, 529, 531] payload_rows_per_dst=[5468, 5570, 5456, 5518, 5626, 5632, 5512, 5658] dup_rows_per_dst=[2616, 2590, 2488, 2568, 2780, 2628, 2756, 2670] combine_rows_per_src=[5468, 5570, 5456, 5518, 5626, 5632, 5512, 5658] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=229536 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279150752 ingress_bytes=609222656 link_bytes=609222656 token_bytes=558071968 weight_movement_bytes=330301440 total_nvlink_bytes=888373408 weight_to_token_ratio=0.591862 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059538399e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38912 dup_rows=21068 dup_rows_remote=18472 dedup_ratio=0.321472 router_tokens_per_expert=[494, 500, 495, 501, 535, 509, 516, 512, 525, 495, 511, 525, 495, 494, 527, 461, 530, 519, 547, 511, 524, 511, 513, 546, 507, 512, 492, 481, 518, 505, 554, 534, 511, 497, 520, 469, 512, 506, 513, 548, 517, 508, 511, 523, 539, 493, 536, 513, 538, 514, 500, 544, 502, 520, 495, 471, 558, 453, 506, 534, 527, 500, 486, 505] payload_rows_per_dst=[5542, 5508, 5630, 5620, 5556, 5486, 5602, 5524] dup_rows_per_dst=[2582, 2558, 2772, 2586, 2596, 2794, 2566, 2614] combine_rows_per_src=[5542, 5508, 5630, 5620, 5556, 5486, 5602, 5524] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=228640 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279364896 ingress_bytes=609437696 link_bytes=609437696 token_bytes=558501152 weight_movement_bytes=330301440 total_nvlink_bytes=888802592 weight_to_token_ratio=0.591407 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059912389e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=39102 dup_rows=20938 dup_rows_remote=18366 dedup_ratio=0.319489 router_tokens_per_expert=[504, 543, 509, 512, 530, 543, 508, 492, 533, 525, 535, 514, 523, 498, 512, 524, 481, 523, 506, 499, 530, 518, 515, 518, 478, 531, 521, 515, 525, 520, 518, 530, 530, 499, 491, 513, 481, 541, 531, 539, 502, 515, 494, 514, 489, 497, 501, 522, 511, 468, 519, 487, 507, 524, 527, 502, 509, 507, 509, 502, 507, 516, 497, 484] payload_rows_per_dst=[5592, 5684, 5542, 5612, 5648, 5496, 5550, 5474] dup_rows_per_dst=[2690, 2644, 2638, 2664, 2602, 2572, 2540, 2588] combine_rows_per_src=[5592, 5684, 5542, 5612, 5648, 5496, 5550, 5474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38880 dup_rows=21088 dup_rows_remote=18446 dedup_ratio=0.321777 router_tokens_per_expert=[503, 497, 514, 528, 507, 505, 476, 501, 519, 509, 527, 493, 471, 490, 537, 548, 539, 480, 531, 520, 524, 489, 537, 507, 509, 514, 510, 523, 548, 491, 499, 511, 546, 490, 543, 489, 504, 501, 503, 503, 496, 526, 507, 508, 533, 496, 506, 519, 499, 528, 520, 516, 511, 517, 507, 530, 495, 491, 484, 520, 544, 528, 520, 531] payload_rows_per_dst=[5514, 5554, 5604, 5568, 5524, 5636, 5474, 5574] dup_rows_per_dst=[2548, 2634, 2650, 2642, 2634, 2546, 2782, 2652] combine_rows_per_src=[5514, 5554, 5604, 5568, 5524, 5636, 5474, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280111104 dispatch_meta_bytes=229952 combine_payload_bytes=280111104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280341056 ingress_bytes=390211584 link_bytes=390211584 token_bytes=560452160 weight_movement_bytes=110100480 total_nvlink_bytes=670552640 weight_to_token_ratio=0.196449 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786421234e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=229872 combine_payload_bytes=280283136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280513008 ingress_bytes=390383616 link_bytes=390383616 token_bytes=560796144 weight_movement_bytes=110100480 total_nvlink_bytes=670896624 weight_to_token_ratio=0.196329 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789413153e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44440 payload_rows_remote=38984 dup_rows=21096 dup_rows_remote=18608 dedup_ratio=0.321899 router_tokens_per_expert=[510, 503, 529, 483, 520, 522, 492, 483, 514, 519, 546, 472, 518, 523, 491, 497, 481, 519, 467, 477, 485, 503, 521, 519, 498, 493, 476, 507, 547, 472, 525, 525, 541, 492, 551, 511, 524, 536, 520, 528, 535, 514, 503, 493, 547, 532, 491, 515, 531, 523, 481, 503, 500, 497, 531, 568, 538, 512, 543, 494, 514, 503, 529, 531] payload_rows_per_dst=[5468, 5570, 5456, 5518, 5626, 5632, 5512, 5658] dup_rows_per_dst=[2616, 2590, 2488, 2568, 2780, 2628, 2756, 2670] combine_rows_per_src=[5468, 5570, 5456, 5518, 5626, 5632, 5512, 5658] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277272576 dispatch_meta_bytes=228400 combine_payload_bytes=277272576 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277500976 ingress_bytes=387373056 link_bytes=387373056 token_bytes=554773552 weight_movement_bytes=110100480 total_nvlink_bytes=664874032 weight_to_token_ratio=0.198460 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.737054563e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229304 combine_payload_bytes=278691840 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278921144 ingress_bytes=498892800 link_bytes=498892800 token_bytes=557612984 weight_movement_bytes=220200960 total_nvlink_bytes=777813944 weight_to_token_ratio=0.394899 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676566330e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38894 dup_rows=20966 dup_rows_remote=18168 dedup_ratio=0.319916 router_tokens_per_expert=[495, 541, 476, 507, 486, 502, 535, 530, 516, 525, 498, 511, 531, 520, 513, 538, 488, 514, 520, 505, 572, 486, 526, 540, 515, 497, 522, 500, 488, 485, 538, 512, 523, 507, 496, 492, 504, 518, 523, 525, 518, 476, 528, 540, 484, 521, 484, 490, 491, 513, 540, 527, 525, 542, 510, 492, 493, 518, 523, 511, 475, 523, 508, 516] payload_rows_per_dst=[5676, 5468, 5570, 5562, 5580, 5506, 5644, 5564] dup_rows_per_dst=[2798, 2508, 2784, 2550, 2544, 2576, 2636, 2570] combine_rows_per_src=[5676, 5468, 5570, 5562, 5580, 5506, 5644, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=230368 combine_payload_bytes=279437312 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279667680 ingress_bytes=499638272 link_bytes=499638272 token_bytes=559104992 weight_movement_bytes=220200960 total_nvlink_bytes=779305952 weight_to_token_ratio=0.393845 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689531314e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=229872 combine_payload_bytes=280283136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280513008 ingress_bytes=390383616 link_bytes=390383616 token_bytes=560796144 weight_movement_bytes=110100480 total_nvlink_bytes=670896624 weight_to_token_ratio=0.196329 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789413153e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=229536 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279150752 ingress_bytes=609222656 link_bytes=609222656 token_bytes=558071968 weight_movement_bytes=330301440 total_nvlink_bytes=888373408 weight_to_token_ratio=0.591862 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059538399e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280111104 dispatch_meta_bytes=229952 combine_payload_bytes=280111104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280341056 ingress_bytes=390211584 link_bytes=390211584 token_bytes=560452160 weight_movement_bytes=110100480 total_nvlink_bytes=670552640 weight_to_token_ratio=0.196449 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.786421234e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38942 dup_rows=20984 dup_rows_remote=18218 dedup_ratio=0.320190 router_tokens_per_expert=[534, 489, 501, 521, 497, 505, 498, 533, 506, 484, 533, 551, 552, 537, 519, 506, 507, 510, 496, 478, 537, 545, 541, 515, 508, 530, 551, 547, 487, 537, 498, 498, 538, 505, 512, 476, 500, 516, 496, 520, 465, 504, 504, 531, 514, 494, 541, 508, 508, 495, 492, 529, 534, 460, 500, 515, 536, 499, 517, 470, 509, 527, 480, 522] payload_rows_per_dst=[5588, 5610, 5636, 5584, 5548, 5486, 5542, 5558] dup_rows_per_dst=[2568, 2766, 2622, 2728, 2578, 2636, 2524, 2562] combine_rows_per_src=[5588, 5610, 5636, 5584, 5548, 5486, 5542, 5558] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=229304 combine_payload_bytes=278691840 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278921144 ingress_bytes=498892800 link_bytes=498892800 token_bytes=557612984 weight_movement_bytes=220200960 total_nvlink_bytes=777813944 weight_to_token_ratio=0.394899 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676566330e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=230368 combine_payload_bytes=279437312 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279667680 ingress_bytes=499638272 link_bytes=499638272 token_bytes=559104992 weight_movement_bytes=220200960 total_nvlink_bytes=779305952 weight_to_token_ratio=0.393845 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.689531314e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=228248 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279020440 ingress_bytes=388892672 link_bytes=388892672 token_bytes=557812632 weight_movement_bytes=110100480 total_nvlink_bytes=667913112 weight_to_token_ratio=0.197379 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.763483185e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38776 dup_rows=21146 dup_rows_remote=18428 dedup_ratio=0.322662 router_tokens_per_expert=[488, 519, 517, 532, 521, 508, 500, 543, 518, 507, 544, 498, 518, 546, 542, 550, 545, 519, 503, 502, 492, 506, 502, 507, 475, 500, 499, 506, 507, 563, 546, 509, 502, 549, 550, 535, 533, 492, 455, 516, 482, 537, 529, 537, 536, 534, 512, 499, 526, 481, 487, 526, 488, 481, 494, 503, 503, 483, 506, 511, 495, 485, 481, 488] payload_rows_per_dst=[5556, 5646, 5508, 5592, 5590, 5614, 5462, 5422] dup_rows_per_dst=[2700, 2800, 2644, 2618, 2674, 2718, 2510, 2482] combine_rows_per_src=[5556, 5646, 5508, 5592, 5590, 5614, 5462, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=228640 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279364896 ingress_bytes=609437696 link_bytes=609437696 token_bytes=558501152 weight_movement_bytes=330301440 total_nvlink_bytes=888802592 weight_to_token_ratio=0.591407 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059912389e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38812 dup_rows=21114 dup_rows_remote=18504 dedup_ratio=0.322174 router_tokens_per_expert=[541, 567, 494, 544, 510, 497, 487, 510, 515, 549, 529, 558, 513, 501, 487, 500, 496, 524, 499, 493, 542, 488, 556, 474, 501, 525, 529, 516, 485, 505, 491, 521, 512, 499, 514, 490, 550, 518, 524, 466, 514, 512, 512, 516, 527, 511, 511, 523, 524, 508, 512, 514, 484, 467, 517, 509, 467, 506, 508, 556, 486, 523, 524, 517] payload_rows_per_dst=[5334, 5560, 5570, 5522, 5510, 5628, 5688, 5610] dup_rows_per_dst=[2440, 2744, 2574, 2624, 2590, 2624, 2908, 2610] combine_rows_per_src=[5334, 5560, 5570, 5522, 5510, 5628, 5688, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44686 payload_rows_remote=39116 dup_rows=20850 dup_rows_remote=18318 dedup_ratio=0.318146 router_tokens_per_expert=[489, 492, 494, 563, 500, 466, 537, 531, 514, 538, 498, 533, 515, 468, 495, 516, 521, 517, 543, 503, 482, 525, 494, 530, 522, 503, 491, 493, 499, 520, 536, 487, 462, 501, 552, 517, 505, 534, 521, 496, 497, 526, 497, 547, 492, 515, 528, 521, 511, 510, 505, 517, 504, 525, 479, 529, 533, 529, 509, 533, 524, 502, 513, 519] payload_rows_per_dst=[5578, 5580, 5606, 5570, 5578, 5556, 5590, 5628] dup_rows_per_dst=[2566, 2574, 2624, 2532, 2598, 2690, 2570, 2696] combine_rows_per_src=[5578, 5580, 5606, 5570, 5578, 5556, 5590, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44430 payload_rows_remote=38830 dup_rows=21106 dup_rows_remote=18448 dedup_ratio=0.322052 router_tokens_per_expert=[503, 482, 516, 536, 524, 566, 507, 498, 514, 542, 530, 501, 505, 522, 478, 488, 499, 480, 539, 505, 507, 504, 567, 510, 503, 521, 509, 496, 506, 533, 522, 494, 558, 540, 492, 546, 488, 508, 484, 490, 528, 521, 498, 531, 471, 515, 485, 507, 534, 538, 490, 498, 515, 542, 496, 516, 507, 530, 519, 445, 552, 504, 489, 524] payload_rows_per_dst=[5570, 5528, 5540, 5566, 5572, 5516, 5600, 5538] dup_rows_per_dst=[2694, 2632, 2682, 2602, 2640, 2596, 2658, 2602] combine_rows_per_src=[5570, 5528, 5540, 5566, 5572, 5516, 5600, 5538] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44462 payload_rows_remote=38842 dup_rows=21074 dup_rows_remote=18406 dedup_ratio=0.321564 router_tokens_per_expert=[545, 503, 488, 523, 553, 526, 488, 536, 512, 507, 546, 503, 507, 511, 524, 467, 502, 513, 477, 520, 555, 486, 506, 512, 546, 465, 537, 481, 522, 551, 522, 528, 501, 509, 505, 515, 534, 526, 552, 502, 537, 513, 501, 509, 529, 517, 503, 471, 519, 480, 504, 516, 495, 492, 511, 502, 515, 530, 509, 491, 496, 529, 479, 514] payload_rows_per_dst=[5600, 5514, 5548, 5560, 5620, 5580, 5502, 5538] dup_rows_per_dst=[2724, 2640, 2594, 2744, 2668, 2580, 2536, 2588] combine_rows_per_src=[5600, 5514, 5548, 5560, 5620, 5580, 5502, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44514 payload_rows_remote=39018 dup_rows=21022 dup_rows_remote=18346 dedup_ratio=0.320770 router_tokens_per_expert=[545, 522, 497, 540, 477, 531, 489, 512, 513, 519, 494, 504, 507, 489, 541, 519, 524, 494, 506, 517, 529, 487, 509, 545, 524, 506, 541, 538, 496, 498, 485, 515, 500, 462, 483, 547, 532, 528, 475, 541, 536, 495, 528, 533, 522, 478, 486, 475, 485, 494, 539, 514, 522, 485, 531, 515, 514, 536, 511, 521, 555, 478, 528, 506] payload_rows_per_dst=[5590, 5496, 5584, 5500, 5556, 5566, 5600, 5622] dup_rows_per_dst=[2636, 2676, 2638, 2706, 2580, 2540, 2570, 2676] combine_rows_per_src=[5590, 5496, 5584, 5500, 5556, 5566, 5600, 5622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228816 combine_payload_bytes=277946368 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278175184 ingress_bytes=608247808 link_bytes=608247808 token_bytes=556121552 weight_movement_bytes=330301440 total_nvlink_bytes=886422992 weight_to_token_ratio=0.593937 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057842978e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44430 payload_rows_remote=38830 dup_rows=21106 dup_rows_remote=18448 dedup_ratio=0.322052 router_tokens_per_expert=[503, 482, 516, 536, 524, 566, 507, 498, 514, 542, 530, 501, 505, 522, 478, 488, 499, 480, 539, 505, 507, 504, 567, 510, 503, 521, 509, 496, 506, 533, 522, 494, 558, 540, 492, 546, 488, 508, 484, 490, 528, 521, 498, 531, 471, 515, 485, 507, 534, 538, 490, 498, 515, 542, 496, 516, 507, 530, 519, 445, 552, 504, 489, 524] payload_rows_per_dst=[5570, 5528, 5540, 5566, 5572, 5516, 5600, 5538] dup_rows_per_dst=[2694, 2632, 2682, 2602, 2640, 2596, 2658, 2602] combine_rows_per_src=[5570, 5528, 5540, 5566, 5572, 5516, 5600, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38812 dup_rows=21114 dup_rows_remote=18504 dedup_ratio=0.322174 router_tokens_per_expert=[541, 567, 494, 544, 510, 497, 487, 510, 515, 549, 529, 558, 513, 501, 487, 500, 496, 524, 499, 493, 542, 488, 556, 474, 501, 525, 529, 516, 485, 505, 491, 521, 512, 499, 514, 490, 550, 518, 524, 466, 514, 512, 512, 516, 527, 511, 511, 523, 524, 508, 512, 514, 484, 467, 517, 509, 467, 506, 508, 556, 486, 523, 524, 517] payload_rows_per_dst=[5334, 5560, 5570, 5522, 5510, 5628, 5688, 5610] dup_rows_per_dst=[2440, 2744, 2574, 2624, 2590, 2624, 2908, 2610] combine_rows_per_src=[5334, 5560, 5570, 5522, 5510, 5628, 5688, 5610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229456 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279910480 ingress_bytes=499881984 link_bytes=499881984 token_bytes=559591504 weight_movement_bytes=220200960 total_nvlink_bytes=779792464 weight_to_token_ratio=0.393503 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693769866e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=228992 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278648448 ingress_bytes=498620416 link_bytes=498620416 token_bytes=557067904 weight_movement_bytes=220200960 total_nvlink_bytes=777268864 weight_to_token_ratio=0.395286 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671829124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278204416 dispatch_meta_bytes=229264 combine_payload_bytes=278204416 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278433680 ingress_bytes=498405376 link_bytes=498405376 token_bytes=556638096 weight_movement_bytes=220200960 total_nvlink_bytes=776839056 weight_to_token_ratio=0.395591 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.668089225e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44524 payload_rows_remote=39018 dup_rows=21012 dup_rows_remote=18450 dedup_ratio=0.320618 router_tokens_per_expert=[504, 532, 534, 515, 488, 541, 519, 546, 505, 520, 497, 523, 514, 526, 544, 515, 513, 529, 479, 522, 506, 514, 451, 520, 518, 484, 484, 487, 519, 514, 492, 562, 508, 527, 491, 524, 496, 518, 523, 528, 498, 505, 498, 492, 546, 537, 495, 542, 517, 514, 506, 527, 482, 524, 500, 506, 512, 498, 487, 495, 493, 516, 511, 535] payload_rows_per_dst=[5656, 5630, 5506, 5480, 5586, 5604, 5572, 5490] dup_rows_per_dst=[2702, 2658, 2562, 2640, 2644, 2622, 2580, 2604] combine_rows_per_src=[5656, 5630, 5506, 5480, 5586, 5604, 5572, 5490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38776 dup_rows=21146 dup_rows_remote=18428 dedup_ratio=0.322662 router_tokens_per_expert=[488, 519, 517, 532, 521, 508, 500, 543, 518, 507, 544, 498, 518, 546, 542, 550, 545, 519, 503, 502, 492, 506, 502, 507, 475, 500, 499, 506, 507, 563, 546, 509, 502, 549, 550, 535, 533, 492, 455, 516, 482, 537, 529, 537, 536, 534, 512, 499, 526, 481, 487, 526, 488, 481, 494, 503, 503, 483, 506, 511, 495, 485, 481, 488] payload_rows_per_dst=[5556, 5646, 5508, 5592, 5590, 5614, 5462, 5422] dup_rows_per_dst=[2700, 2800, 2644, 2618, 2674, 2718, 2510, 2482] combine_rows_per_src=[5556, 5646, 5508, 5592, 5590, 5614, 5462, 5422] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44686 payload_rows_remote=39116 dup_rows=20850 dup_rows_remote=18318 dedup_ratio=0.318146 router_tokens_per_expert=[489, 492, 494, 563, 500, 466, 537, 531, 514, 538, 498, 533, 515, 468, 495, 516, 521, 517, 543, 503, 482, 525, 494, 530, 522, 503, 491, 493, 499, 520, 536, 487, 462, 501, 552, 517, 505, 534, 521, 496, 497, 526, 497, 547, 492, 515, 528, 521, 511, 510, 505, 517, 504, 525, 479, 529, 533, 529, 509, 533, 524, 502, 513, 519] payload_rows_per_dst=[5578, 5580, 5606, 5570, 5578, 5556, 5590, 5628] dup_rows_per_dst=[2566, 2574, 2624, 2532, 2598, 2690, 2570, 2696] combine_rows_per_src=[5578, 5580, 5606, 5570, 5578, 5556, 5590, 5628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280383488 dispatch_meta_bytes=229736 combine_payload_bytes=280383488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280613224 ingress_bytes=390483968 link_bytes=390483968 token_bytes=560996712 weight_movement_bytes=110100480 total_nvlink_bytes=671097192 weight_to_token_ratio=0.196259 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.791158439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=229112 combine_payload_bytes=278333440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278562552 ingress_bytes=388433920 link_bytes=388433920 token_bytes=556895992 weight_movement_bytes=110100480 total_nvlink_bytes=666996472 weight_to_token_ratio=0.197704 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.755504733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=228992 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278648448 ingress_bytes=498620416 link_bytes=498620416 token_bytes=557067904 weight_movement_bytes=220200960 total_nvlink_bytes=777268864 weight_to_token_ratio=0.395286 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671829124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44306 payload_rows_remote=38782 dup_rows=21230 dup_rows_remote=18564 dedup_ratio=0.323944 router_tokens_per_expert=[482, 541, 504, 470, 479, 508, 524, 499, 484, 499, 506, 503, 488, 541, 499, 495, 528, 513, 498, 506, 514, 515, 545, 495, 530, 499, 512, 505, 501, 513, 515, 497, 478, 486, 511, 528, 530, 491, 510, 524, 506, 508, 509, 477, 545, 517, 551, 515, 525, 511, 499, 490, 514, 517, 512, 527, 554, 527, 510, 552, 517, 560, 522, 537] payload_rows_per_dst=[5524, 5398, 5604, 5614, 5492, 5470, 5542, 5662] dup_rows_per_dst=[2666, 2460, 2752, 2666, 2638, 2632, 2648, 2768] combine_rows_per_src=[5524, 5398, 5604, 5614, 5492, 5470, 5542, 5662] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278204416 dispatch_meta_bytes=229264 combine_payload_bytes=278204416 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278433680 ingress_bytes=498405376 link_bytes=498405376 token_bytes=556638096 weight_movement_bytes=220200960 total_nvlink_bytes=776839056 weight_to_token_ratio=0.395591 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.668089225e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229872 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279910896 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591920 weight_movement_bytes=110100480 total_nvlink_bytes=669692400 weight_to_token_ratio=0.196751 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228816 combine_payload_bytes=277946368 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278175184 ingress_bytes=608247808 link_bytes=608247808 token_bytes=556121552 weight_movement_bytes=330301440 total_nvlink_bytes=886422992 weight_to_token_ratio=0.593937 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057842978e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44514 payload_rows_remote=39018 dup_rows=21022 dup_rows_remote=18346 dedup_ratio=0.320770 router_tokens_per_expert=[545, 522, 497, 540, 477, 531, 489, 512, 513, 519, 494, 504, 507, 489, 541, 519, 524, 494, 506, 517, 529, 487, 509, 545, 524, 506, 541, 538, 496, 498, 485, 515, 500, 462, 483, 547, 532, 528, 475, 541, 536, 495, 528, 533, 522, 478, 486, 475, 485, 494, 539, 514, 522, 485, 531, 515, 514, 536, 511, 521, 555, 478, 528, 506] payload_rows_per_dst=[5590, 5496, 5584, 5500, 5556, 5566, 5600, 5622] dup_rows_per_dst=[2636, 2676, 2638, 2706, 2580, 2540, 2570, 2676] combine_rows_per_src=[5590, 5496, 5584, 5500, 5556, 5566, 5600, 5622] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280383488 dispatch_meta_bytes=229736 combine_payload_bytes=280383488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280613224 ingress_bytes=390483968 link_bytes=390483968 token_bytes=560996712 weight_movement_bytes=110100480 total_nvlink_bytes=671097192 weight_to_token_ratio=0.196259 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.791158439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229872 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279910896 ingress_bytes=389781504 link_bytes=389781504 token_bytes=559591920 weight_movement_bytes=110100480 total_nvlink_bytes=669692400 weight_to_token_ratio=0.196751 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778941435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277989376 dispatch_meta_bytes=229384 combine_payload_bytes=277989376 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278218760 ingress_bytes=498190336 link_bytes=498190336 token_bytes=556208136 weight_movement_bytes=220200960 total_nvlink_bytes=776409096 weight_to_token_ratio=0.395897 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.664349325e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44456 payload_rows_remote=38944 dup_rows=21080 dup_rows_remote=18484 dedup_ratio=0.321655 router_tokens_per_expert=[480, 525, 514, 505, 537, 515, 479, 517, 526, 479, 523, 554, 498, 504, 471, 499, 528, 538, 522, 535, 473, 489, 477, 512, 514, 529, 514, 539, 524, 505, 507, 543, 509, 488, 562, 497, 518, 502, 514, 563, 509, 550, 510, 522, 521, 494, 490, 490, 501, 511, 526, 475, 552, 514, 494, 490, 523, 493, 505, 553, 515, 505, 508, 489] payload_rows_per_dst=[5572, 5512, 5550, 5690, 5586, 5518, 5468, 5560] dup_rows_per_dst=[2572, 2596, 2598, 2660, 2720, 2654, 2658, 2622] combine_rows_per_src=[5572, 5512, 5550, 5690, 5586, 5518, 5468, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38850 dup_rows=21120 dup_rows_remote=18454 dedup_ratio=0.322266 router_tokens_per_expert=[530, 533, 479, 518, 557, 506, 508, 516, 498, 520, 536, 531, 515, 469, 511, 517, 523, 483, 509, 495, 500, 506, 538, 533, 480, 508, 464, 507, 487, 525, 500, 512, 480, 516, 493, 562, 490, 523, 525, 527, 519, 544, 495, 544, 483, 522, 539, 565, 472, 473, 521, 523, 461, 522, 468, 521, 530, 522, 515, 510, 510, 527, 503, 549] payload_rows_per_dst=[5576, 5508, 5590, 5476, 5566, 5684, 5486, 5530] dup_rows_per_dst=[2718, 2686, 2584, 2490, 2666, 2738, 2436, 2802] combine_rows_per_src=[5576, 5508, 5590, 5476, 5566, 5684, 5486, 5530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=39098 dup_rows=20946 dup_rows_remote=18454 dedup_ratio=0.319611 router_tokens_per_expert=[479, 490, 487, 496, 515, 509, 530, 511, 538, 489, 532, 530, 524, 476, 558, 507, 538, 508, 508, 508, 507, 513, 509, 524, 486, 517, 534, 544, 504, 493, 520, 538, 540, 527, 511, 515, 526, 503, 544, 506, 523, 511, 515, 497, 512, 531, 493, 493, 495, 495, 502, 473, 488, 513, 517, 509, 489, 535, 537, 505, 489, 528, 512, 512] payload_rows_per_dst=[5488, 5596, 5606, 5554, 5620, 5640, 5492, 5594] dup_rows_per_dst=[2546, 2712, 2624, 2718, 2724, 2510, 2492, 2620] combine_rows_per_src=[5488, 5596, 5606, 5554, 5620, 5640, 5492, 5594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=229712 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279380304 ingress_bytes=609452032 link_bytes=609452032 token_bytes=558530896 weight_movement_bytes=330301440 total_nvlink_bytes=888832336 weight_to_token_ratio=0.591375 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059937321e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=229216 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278706016 ingress_bytes=498677760 link_bytes=498677760 token_bytes=557182816 weight_movement_bytes=220200960 total_nvlink_bytes=777383776 weight_to_token_ratio=0.395204 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672826430e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38908 dup_rows=21046 dup_rows_remote=18498 dedup_ratio=0.321136 router_tokens_per_expert=[526, 495, 536, 510, 523, 524, 513, 496, 515, 492, 510, 527, 494, 535, 532, 494, 541, 494, 493, 511, 504, 535, 548, 499, 485, 493, 502, 509, 521, 513, 508, 508, 495, 546, 563, 543, 508, 456, 503, 502, 492, 530, 518, 503, 536, 493, 535, 527, 531, 496, 496, 516, 510, 484, 509, 525, 512, 498, 507, 525, 533, 499, 493, 498] payload_rows_per_dst=[5538, 5498, 5540, 5558, 5454, 5606, 5714, 5582] dup_rows_per_dst=[2708, 2644, 2684, 2520, 2456, 2718, 2768, 2548] combine_rows_per_src=[5538, 5498, 5540, 5558, 5454, 5606, 5714, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44400 payload_rows_remote=38882 dup_rows=21136 dup_rows_remote=18516 dedup_ratio=0.322510 router_tokens_per_expert=[508, 483, 501, 496, 502, 521, 530, 516, 526, 532, 510, 491, 493, 513, 548, 480, 510, 504, 513, 539, 502, 519, 520, 510, 533, 496, 489, 476, 509, 481, 514, 487, 517, 503, 517, 498, 512, 559, 498, 495, 499, 484, 527, 502, 498, 489, 513, 557, 500, 523, 558, 545, 543, 520, 513, 525, 538, 495, 509, 524, 509, 518, 505, 523] payload_rows_per_dst=[5510, 5586, 5482, 5450, 5598, 5518, 5656, 5600] dup_rows_per_dst=[2604, 2600, 2752, 2520, 2600, 2620, 2798, 2642] combine_rows_per_src=[5510, 5586, 5482, 5450, 5598, 5518, 5656, 5600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44810 payload_rows_remote=39212 dup_rows=20726 dup_rows_remote=18086 dedup_ratio=0.316254 router_tokens_per_expert=[514, 510, 558, 485, 561, 473, 464, 504, 501, 505, 485, 548, 477, 533, 521, 515, 517, 529, 481, 536, 524, 477, 500, 555, 487, 531, 513, 511, 493, 535, 470, 505, 513, 526, 514, 535, 499, 531, 541, 502, 527, 548, 494, 511, 487, 480, 504, 548, 504, 528, 546, 531, 524, 497, 488, 524, 483, 531, 537, 508, 524, 491, 473, 501] payload_rows_per_dst=[5534, 5606, 5598, 5600, 5622, 5622, 5622, 5606] dup_rows_per_dst=[2604, 2564, 2640, 2490, 2700, 2576, 2662, 2490] combine_rows_per_src=[5534, 5606, 5598, 5600, 5622, 5622, 5622, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279681024 dispatch_meta_bytes=229456 combine_payload_bytes=279681024 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279910480 ingress_bytes=499881984 link_bytes=499881984 token_bytes=559591504 weight_movement_bytes=220200960 total_nvlink_bytes=779792464 weight_to_token_ratio=0.393503 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693769866e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38908 dup_rows=21046 dup_rows_remote=18498 dedup_ratio=0.321136 router_tokens_per_expert=[526, 495, 536, 510, 523, 524, 513, 496, 515, 492, 510, 527, 494, 535, 532, 494, 541, 494, 493, 511, 504, 535, 548, 499, 485, 493, 502, 509, 521, 513, 508, 508, 495, 546, 563, 543, 508, 456, 503, 502, 492, 530, 518, 503, 536, 493, 535, 527, 531, 496, 496, 516, 510, 484, 509, 525, 512, 498, 507, 525, 533, 499, 493, 498] payload_rows_per_dst=[5538, 5498, 5540, 5558, 5454, 5606, 5714, 5582] dup_rows_per_dst=[2708, 2644, 2684, 2520, 2456, 2718, 2768, 2548] combine_rows_per_src=[5538, 5498, 5540, 5558, 5454, 5606, 5714, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44378 payload_rows_remote=38804 dup_rows=21158 dup_rows_remote=18268 dedup_ratio=0.322845 router_tokens_per_expert=[511, 546, 526, 513, 524, 519, 508, 516, 500, 517, 490, 516, 511, 508, 521, 552, 505, 512, 504, 527, 518, 507, 492, 491, 525, 513, 507, 535, 505, 531, 556, 560, 483, 515, 520, 475, 517, 484, 458, 515, 531, 519, 486, 500, 520, 511, 518, 489, 525, 469, 552, 530, 503, 512, 487, 500, 494, 492, 561, 530, 482, 499, 529, 496] payload_rows_per_dst=[5618, 5562, 5534, 5574, 5446, 5578, 5580, 5486] dup_rows_per_dst=[2708, 2668, 2578, 2890, 2488, 2570, 2576, 2680] combine_rows_per_src=[5618, 5562, 5534, 5574, 5446, 5578, 5580, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=39098 dup_rows=20946 dup_rows_remote=18454 dedup_ratio=0.319611 router_tokens_per_expert=[479, 490, 487, 496, 515, 509, 530, 511, 538, 489, 532, 530, 524, 476, 558, 507, 538, 508, 508, 508, 507, 513, 509, 524, 486, 517, 534, 544, 504, 493, 520, 538, 540, 527, 511, 515, 526, 503, 544, 506, 523, 511, 515, 497, 512, 531, 493, 493, 495, 495, 502, 473, 488, 513, 517, 509, 489, 535, 537, 505, 489, 528, 512, 512] payload_rows_per_dst=[5488, 5596, 5606, 5554, 5620, 5640, 5492, 5594] dup_rows_per_dst=[2546, 2712, 2624, 2718, 2724, 2510, 2492, 2620] combine_rows_per_src=[5488, 5596, 5606, 5554, 5620, 5640, 5492, 5594] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280254464 dispatch_meta_bytes=230208 combine_payload_bytes=280254464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280484672 ingress_bytes=390354944 link_bytes=390354944 token_bytes=560739136 weight_movement_bytes=110100480 total_nvlink_bytes=670839616 weight_to_token_ratio=0.196349 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788914500e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38850 dup_rows=21120 dup_rows_remote=18454 dedup_ratio=0.322266 router_tokens_per_expert=[530, 533, 479, 518, 557, 506, 508, 516, 498, 520, 536, 531, 515, 469, 511, 517, 523, 483, 509, 495, 500, 506, 538, 533, 480, 508, 464, 507, 487, 525, 500, 512, 480, 516, 493, 562, 490, 523, 525, 527, 519, 544, 495, 544, 483, 522, 539, 565, 472, 473, 521, 523, 461, 522, 468, 521, 530, 522, 515, 510, 510, 527, 503, 549] payload_rows_per_dst=[5576, 5508, 5590, 5476, 5566, 5684, 5486, 5530] dup_rows_per_dst=[2718, 2686, 2584, 2490, 2666, 2738, 2436, 2802] combine_rows_per_src=[5576, 5508, 5590, 5476, 5566, 5684, 5486, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=229592 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278935768 ingress_bytes=498907136 link_bytes=498907136 token_bytes=557641944 weight_movement_bytes=220200960 total_nvlink_bytes=777842904 weight_to_token_ratio=0.394879 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676815656e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44622 payload_rows_remote=38978 dup_rows=20914 dup_rows_remote=18318 dedup_ratio=0.319122 router_tokens_per_expert=[567, 509, 495, 476, 520, 511, 508, 476, 532, 502, 485, 505, 548, 524, 514, 539, 489, 502, 537, 516, 492, 555, 497, 506, 481, 493, 516, 534, 510, 494, 514, 483, 500, 561, 527, 516, 513, 525, 512, 491, 511, 503, 540, 504, 529, 507, 530, 516, 487, 525, 517, 510, 511, 543, 486, 485, 526, 518, 469, 533, 483, 504, 532, 524] payload_rows_per_dst=[5644, 5554, 5574, 5646, 5534, 5594, 5516, 5560] dup_rows_per_dst=[2596, 2644, 2602, 2700, 2756, 2390, 2612, 2614] combine_rows_per_src=[5644, 5554, 5574, 5646, 5534, 5594, 5516, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278892544 dispatch_meta_bytes=229624 combine_payload_bytes=278892544 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279122168 ingress_bytes=388993024 link_bytes=388993024 token_bytes=558014712 weight_movement_bytes=110100480 total_nvlink_bytes=668115192 weight_to_token_ratio=0.197307 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.051471 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765228471e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278706176 dispatch_meta_bytes=229592 combine_payload_bytes=278706176 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278935768 ingress_bytes=498907136 link_bytes=498907136 token_bytes=557641944 weight_movement_bytes=220200960 total_nvlink_bytes=777842904 weight_to_token_ratio=0.394879 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676815656e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281071616 dispatch_meta_bytes=229192 combine_payload_bytes=281071616 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=281300808 ingress_bytes=611373056 link_bytes=611373056 token_bytes=562372424 weight_movement_bytes=330301440 total_nvlink_bytes=892673864 weight_to_token_ratio=0.587336 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.063278298e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=228288 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278375360 ingress_bytes=388247552 link_bytes=388247552 token_bytes=556522432 weight_movement_bytes=110100480 total_nvlink_bytes=666622912 weight_to_token_ratio=0.197837 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752263487e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44488 payload_rows_remote=39006 dup_rows=21048 dup_rows_remote=18486 dedup_ratio=0.321167 router_tokens_per_expert=[495, 535, 499, 505, 506, 532, 497, 515, 505, 526, 481, 501, 515, 478, 510, 506, 509, 489, 526, 496, 499, 552, 516, 547, 522, 465, 521, 510, 501, 487, 488, 549, 529, 530, 489, 536, 485, 532, 536, 492, 509, 508, 513, 491, 512, 527, 552, 505, 504, 473, 557, 520, 501, 514, 499, 507, 502, 504, 558, 506, 539, 525, 494, 536] payload_rows_per_dst=[5580, 5482, 5644, 5570, 5584, 5552, 5516, 5560] dup_rows_per_dst=[2588, 2562, 2624, 2516, 2674, 2682, 2634, 2768] combine_rows_per_src=[5580, 5482, 5644, 5570, 5584, 5552, 5516, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44584 payload_rows_remote=38942 dup_rows=20952 dup_rows_remote=18300 dedup_ratio=0.319702 router_tokens_per_expert=[467, 506, 520, 518, 519, 505, 515, 557, 499, 510, 515, 472, 506, 499, 546, 486, 521, 513, 507, 547, 515, 478, 517, 498, 523, 496, 513, 487, 508, 529, 499, 486, 558, 506, 469, 557, 512, 477, 520, 548, 486, 517, 510, 518, 480, 520, 521, 495, 542, 514, 532, 463, 549, 524, 523, 522, 556, 499, 509, 527, 508, 492, 518, 519] payload_rows_per_dst=[5558, 5524, 5640, 5524, 5642, 5490, 5622, 5584] dup_rows_per_dst=[2656, 2542, 2552, 2558, 2652, 2604, 2716, 2672] combine_rows_per_src=[5558, 5524, 5640, 5524, 5642, 5490, 5622, 5584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=229184 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279623488 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559017792 weight_movement_bytes=110100480 total_nvlink_bytes=669118272 weight_to_token_ratio=0.196953 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44592 payload_rows_remote=38958 dup_rows=20944 dup_rows_remote=18228 dedup_ratio=0.319580 router_tokens_per_expert=[490, 517, 525, 493, 535, 523, 503, 503, 497, 566, 513, 477, 506, 537, 500, 511, 497, 494, 496, 505, 513, 510, 517, 502, 492, 500, 526, 486, 510, 475, 516, 489, 528, 492, 529, 528, 523, 563, 515, 484, 500, 545, 514, 498, 516, 505, 522, 511, 493, 585, 573, 464, 536, 518, 491, 515, 509, 513, 501, 510, 497, 534, 508, 524] payload_rows_per_dst=[5542, 5566, 5566, 5474, 5670, 5626, 5634, 5514] dup_rows_per_dst=[2636, 2648, 2502, 2514, 2654, 2596, 2716, 2678] combine_rows_per_src=[5542, 5566, 5566, 5474, 5670, 5626, 5634, 5514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=229968 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279824976 ingress_bytes=499795968 link_bytes=499795968 token_bytes=559419984 weight_movement_bytes=220200960 total_nvlink_bytes=779620944 weight_to_token_ratio=0.393624 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.692273907e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=228968 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279365224 ingress_bytes=499337216 link_bytes=499337216 token_bytes=558501480 weight_movement_bytes=220200960 total_nvlink_bytes=778702440 weight_to_token_ratio=0.394271 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684295455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44324 payload_rows_remote=38870 dup_rows=21212 dup_rows_remote=18582 dedup_ratio=0.323669 router_tokens_per_expert=[518, 522, 521, 526, 487, 501, 518, 501, 518, 485, 478, 547, 501, 506, 540, 522, 525, 551, 473, 513, 490, 522, 540, 506, 512, 517, 509, 510, 470, 536, 516, 527, 538, 521, 474, 510, 563, 504, 494, 472, 525, 525, 505, 508, 508, 503, 530, 549, 509, 486, 520, 511, 512, 489, 531, 539, 514, 471, 524, 523, 519, 490, 498, 495] payload_rows_per_dst=[5614, 5364, 5528, 5532, 5662, 5612, 5558, 5454] dup_rows_per_dst=[2574, 2420, 2712, 2662, 2900, 2694, 2620, 2630] combine_rows_per_src=[5614, 5364, 5528, 5532, 5662, 5612, 5558, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44810 payload_rows_remote=39212 dup_rows=20726 dup_rows_remote=18086 dedup_ratio=0.316254 router_tokens_per_expert=[514, 510, 558, 485, 561, 473, 464, 504, 501, 505, 485, 548, 477, 533, 521, 515, 517, 529, 481, 536, 524, 477, 500, 555, 487, 531, 513, 511, 493, 535, 470, 505, 513, 526, 514, 535, 499, 531, 541, 502, 527, 548, 494, 511, 487, 480, 504, 548, 504, 528, 546, 531, 524, 497, 488, 524, 483, 531, 537, 508, 524, 491, 473, 501] payload_rows_per_dst=[5534, 5606, 5598, 5600, 5622, 5622, 5622, 5606] dup_rows_per_dst=[2604, 2564, 2640, 2490, 2700, 2576, 2662, 2490] combine_rows_per_src=[5534, 5606, 5598, 5600, 5622, 5622, 5622, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44622 payload_rows_remote=38978 dup_rows=20914 dup_rows_remote=18318 dedup_ratio=0.319122 router_tokens_per_expert=[567, 509, 495, 476, 520, 511, 508, 476, 532, 502, 485, 505, 548, 524, 514, 539, 489, 502, 537, 516, 492, 555, 497, 506, 481, 493, 516, 534, 510, 494, 514, 483, 500, 561, 527, 516, 513, 525, 512, 491, 511, 503, 540, 504, 529, 507, 530, 516, 487, 525, 517, 510, 511, 543, 486, 485, 526, 518, 469, 533, 483, 504, 532, 524] payload_rows_per_dst=[5644, 5554, 5574, 5646, 5534, 5594, 5516, 5560] dup_rows_per_dst=[2596, 2644, 2602, 2700, 2756, 2390, 2612, 2614] combine_rows_per_src=[5644, 5554, 5574, 5646, 5534, 5594, 5516, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280254464 dispatch_meta_bytes=230208 combine_payload_bytes=280254464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280484672 ingress_bytes=390354944 link_bytes=390354944 token_bytes=560739136 weight_movement_bytes=110100480 total_nvlink_bytes=670839616 weight_to_token_ratio=0.196349 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788914500e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44456 payload_rows_remote=38944 dup_rows=21080 dup_rows_remote=18484 dedup_ratio=0.321655 router_tokens_per_expert=[480, 525, 514, 505, 537, 515, 479, 517, 526, 479, 523, 554, 498, 504, 471, 499, 528, 538, 522, 535, 473, 489, 477, 512, 514, 529, 514, 539, 524, 505, 507, 543, 509, 488, 562, 497, 518, 502, 514, 563, 509, 550, 510, 522, 521, 494, 490, 490, 501, 511, 526, 475, 552, 514, 494, 490, 523, 493, 505, 553, 515, 505, 508, 489] payload_rows_per_dst=[5572, 5512, 5550, 5690, 5586, 5518, 5468, 5560] dup_rows_per_dst=[2572, 2596, 2598, 2660, 2720, 2654, 2658, 2622] combine_rows_per_src=[5572, 5512, 5550, 5690, 5586, 5518, 5468, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278476800 dispatch_meta_bytes=229216 combine_payload_bytes=278476800 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278706016 ingress_bytes=498677760 link_bytes=498677760 token_bytes=557182816 weight_movement_bytes=220200960 total_nvlink_bytes=777383776 weight_to_token_ratio=0.395204 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672826430e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281071616 dispatch_meta_bytes=229192 combine_payload_bytes=281071616 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=281300808 ingress_bytes=611373056 link_bytes=611373056 token_bytes=562372424 weight_movement_bytes=330301440 total_nvlink_bytes=892673864 weight_to_token_ratio=0.587336 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.063278298e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=229184 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279623488 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559017792 weight_movement_bytes=110100480 total_nvlink_bytes=669118272 weight_to_token_ratio=0.196953 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=229712 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279380304 ingress_bytes=609452032 link_bytes=609452032 token_bytes=558530896 weight_movement_bytes=330301440 total_nvlink_bytes=888832336 weight_to_token_ratio=0.591375 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059937321e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44352 payload_rows_remote=38758 dup_rows=21184 dup_rows_remote=18488 dedup_ratio=0.323242 router_tokens_per_expert=[517, 540, 509, 495, 499, 517, 493, 504, 533, 514, 530, 525, 500, 538, 562, 545, 535, 502, 484, 502, 481, 521, 515, 540, 516, 515, 518, 514, 495, 507, 511, 500, 516, 500, 514, 471, 495, 505, 547, 528, 489, 512, 523, 519, 534, 504, 502, 562, 559, 459, 456, 484, 500, 537, 462, 503, 515, 510, 519, 530, 513, 522, 494, 507] payload_rows_per_dst=[5494, 5714, 5520, 5538, 5448, 5594, 5464, 5580] dup_rows_per_dst=[2654, 2780, 2640, 2614, 2704, 2696, 2456, 2640] combine_rows_per_src=[5494, 5714, 5520, 5538, 5448, 5594, 5464, 5580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44710 payload_rows_remote=39044 dup_rows=20826 dup_rows_remote=18188 dedup_ratio=0.317780 router_tokens_per_expert=[493, 516, 488, 514, 512, 512, 527, 535, 539, 500, 533, 465, 542, 499, 503, 524, 528, 495, 523, 537, 497, 555, 494, 523, 473, 522, 518, 498, 474, 507, 506, 496, 529, 493, 507, 531, 559, 517, 513, 516, 564, 490, 517, 537, 529, 498, 493, 516, 473, 482, 492, 504, 543, 519, 483, 505, 512, 523, 517, 483, 513, 521, 542, 499] payload_rows_per_dst=[5614, 5590, 5666, 5466, 5600, 5624, 5472, 5678] dup_rows_per_dst=[2580, 2620, 2638, 2522, 2730, 2664, 2530, 2542] combine_rows_per_src=[5614, 5590, 5666, 5466, 5600, 5624, 5472, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44324 payload_rows_remote=38870 dup_rows=21212 dup_rows_remote=18582 dedup_ratio=0.323669 router_tokens_per_expert=[518, 522, 521, 526, 487, 501, 518, 501, 518, 485, 478, 547, 501, 506, 540, 522, 525, 551, 473, 513, 490, 522, 540, 506, 512, 517, 509, 510, 470, 536, 516, 527, 538, 521, 474, 510, 563, 504, 494, 472, 525, 525, 505, 508, 508, 503, 530, 549, 509, 486, 520, 511, 512, 489, 531, 539, 514, 471, 524, 523, 519, 490, 498, 495] payload_rows_per_dst=[5614, 5364, 5528, 5532, 5662, 5612, 5558, 5454] dup_rows_per_dst=[2574, 2420, 2712, 2662, 2900, 2694, 2620, 2630] combine_rows_per_src=[5614, 5364, 5528, 5532, 5662, 5612, 5558, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=39038 dup_rows=20988 dup_rows_remote=18548 dedup_ratio=0.320251 router_tokens_per_expert=[508, 495, 525, 534, 519, 507, 533, 492, 521, 518, 547, 570, 538, 477, 504, 484, 510, 503, 526, 494, 539, 478, 542, 465, 517, 489, 508, 490, 477, 538, 487, 469, 487, 488, 545, 524, 483, 538, 508, 501, 524, 509, 500, 540, 495, 518, 536, 499, 508, 542, 514, 509, 517, 525, 522, 515, 526, 500, 517, 502, 518, 533, 517, 504] payload_rows_per_dst=[5588, 5652, 5494, 5510, 5494, 5550, 5612, 5648] dup_rows_per_dst=[2638, 2666, 2620, 2440, 2654, 2692, 2692, 2586] combine_rows_per_src=[5588, 5652, 5494, 5510, 5494, 5550, 5612, 5648] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=228744 combine_payload_bytes=279250944 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279479688 ingress_bytes=499451904 link_bytes=499451904 token_bytes=558730632 weight_movement_bytes=220200960 total_nvlink_bytes=778931592 weight_to_token_ratio=0.394109 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686290068e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44592 payload_rows_remote=38958 dup_rows=20944 dup_rows_remote=18228 dedup_ratio=0.319580 router_tokens_per_expert=[490, 517, 525, 493, 535, 523, 503, 503, 497, 566, 513, 477, 506, 537, 500, 511, 497, 494, 496, 505, 513, 510, 517, 502, 492, 500, 526, 486, 510, 475, 516, 489, 528, 492, 529, 528, 523, 563, 515, 484, 500, 545, 514, 498, 516, 505, 522, 511, 493, 585, 573, 464, 536, 518, 491, 515, 509, 513, 501, 510, 497, 534, 508, 524] payload_rows_per_dst=[5542, 5566, 5566, 5474, 5670, 5626, 5634, 5514] dup_rows_per_dst=[2636, 2648, 2502, 2514, 2654, 2596, 2716, 2678] combine_rows_per_src=[5542, 5566, 5566, 5474, 5670, 5626, 5634, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44584 payload_rows_remote=38942 dup_rows=20952 dup_rows_remote=18300 dedup_ratio=0.319702 router_tokens_per_expert=[467, 506, 520, 518, 519, 505, 515, 557, 499, 510, 515, 472, 506, 499, 546, 486, 521, 513, 507, 547, 515, 478, 517, 498, 523, 496, 513, 487, 508, 529, 499, 486, 558, 506, 469, 557, 512, 477, 520, 548, 486, 517, 510, 518, 480, 520, 521, 495, 542, 514, 532, 463, 549, 524, 523, 522, 556, 499, 509, 527, 508, 492, 518, 519] payload_rows_per_dst=[5558, 5524, 5640, 5524, 5642, 5490, 5622, 5584] dup_rows_per_dst=[2656, 2542, 2552, 2558, 2652, 2604, 2716, 2672] combine_rows_per_src=[5558, 5524, 5640, 5524, 5642, 5490, 5622, 5584] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38922 dup_rows=21026 dup_rows_remote=18394 dedup_ratio=0.320831 router_tokens_per_expert=[488, 481, 492, 472, 499, 529, 516, 496, 509, 543, 507, 535, 506, 482, 545, 523, 503, 504, 518, 506, 523, 540, 508, 535, 529, 550, 457, 508, 562, 491, 489, 484, 528, 507, 501, 528, 551, 521, 507, 569, 494, 500, 482, 509, 510, 484, 536, 490, 531, 487, 507, 535, 535, 546, 524, 493, 510, 508, 491, 524, 498, 496, 521, 515] payload_rows_per_dst=[5588, 5622, 5586, 5914, 5262, 5500, 5522, 5516] dup_rows_per_dst=[2632, 2784, 2604, 3182, 2206, 2360, 2772, 2486] combine_rows_per_src=[5588, 5622, 5586, 5914, 5262, 5500, 5522, 5516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278620160 dispatch_meta_bytes=229808 combine_payload_bytes=278620160 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278849968 ingress_bytes=498821120 link_bytes=498821120 token_bytes=557470128 weight_movement_bytes=220200960 total_nvlink_bytes=777671088 weight_to_token_ratio=0.395000 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.675319697e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277817344 dispatch_meta_bytes=228984 combine_payload_bytes=277817344 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278046328 ingress_bytes=498018304 link_bytes=498018304 token_bytes=555863672 weight_movement_bytes=220200960 total_nvlink_bytes=776064632 weight_to_token_ratio=0.396142 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661357406e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279867392 dispatch_meta_bytes=228928 combine_payload_bytes=279867392 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280096320 ingress_bytes=500068352 link_bytes=500068352 token_bytes=559963712 weight_movement_bytes=220200960 total_nvlink_bytes=780164672 weight_to_token_ratio=0.393241 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697011112e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278620160 dispatch_meta_bytes=229808 combine_payload_bytes=278620160 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278849968 ingress_bytes=498821120 link_bytes=498821120 token_bytes=557470128 weight_movement_bytes=220200960 total_nvlink_bytes=777671088 weight_to_token_ratio=0.395000 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.675319697e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279824384 dispatch_meta_bytes=230344 combine_payload_bytes=279824384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280054728 ingress_bytes=389924864 link_bytes=389924864 token_bytes=559879112 weight_movement_bytes=110100480 total_nvlink_bytes=669979592 weight_to_token_ratio=0.196650 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781434701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=228744 combine_payload_bytes=279250944 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279479688 ingress_bytes=499451904 link_bytes=499451904 token_bytes=558730632 weight_movement_bytes=220200960 total_nvlink_bytes=778931592 weight_to_token_ratio=0.394109 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686290068e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44510 payload_rows_remote=38922 dup_rows=21026 dup_rows_remote=18394 dedup_ratio=0.320831 router_tokens_per_expert=[488, 481, 492, 472, 499, 529, 516, 496, 509, 543, 507, 535, 506, 482, 545, 523, 503, 504, 518, 506, 523, 540, 508, 535, 529, 550, 457, 508, 562, 491, 489, 484, 528, 507, 501, 528, 551, 521, 507, 569, 494, 500, 482, 509, 510, 484, 536, 490, 531, 487, 507, 535, 535, 546, 524, 493, 510, 508, 491, 524, 498, 496, 521, 515] payload_rows_per_dst=[5588, 5622, 5586, 5914, 5262, 5500, 5522, 5516] dup_rows_per_dst=[2632, 2784, 2604, 3182, 2206, 2360, 2772, 2486] combine_rows_per_src=[5588, 5622, 5586, 5914, 5262, 5500, 5522, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279136256 dispatch_meta_bytes=228968 combine_payload_bytes=279136256 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279365224 ingress_bytes=499337216 link_bytes=499337216 token_bytes=558501480 weight_movement_bytes=220200960 total_nvlink_bytes=778702440 weight_to_token_ratio=0.394271 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684295455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44710 payload_rows_remote=39044 dup_rows=20826 dup_rows_remote=18188 dedup_ratio=0.317780 router_tokens_per_expert=[493, 516, 488, 514, 512, 512, 527, 535, 539, 500, 533, 465, 542, 499, 503, 524, 528, 495, 523, 537, 497, 555, 494, 523, 473, 522, 518, 498, 474, 507, 506, 496, 529, 493, 507, 531, 559, 517, 513, 516, 564, 490, 517, 537, 529, 498, 493, 516, 473, 482, 492, 504, 543, 519, 483, 505, 512, 523, 517, 483, 513, 521, 542, 499] payload_rows_per_dst=[5614, 5590, 5666, 5466, 5600, 5624, 5472, 5678] dup_rows_per_dst=[2580, 2620, 2638, 2522, 2730, 2664, 2530, 2542] combine_rows_per_src=[5614, 5590, 5666, 5466, 5600, 5624, 5472, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44488 payload_rows_remote=39006 dup_rows=21048 dup_rows_remote=18486 dedup_ratio=0.321167 router_tokens_per_expert=[495, 535, 499, 505, 506, 532, 497, 515, 505, 526, 481, 501, 515, 478, 510, 506, 509, 489, 526, 496, 499, 552, 516, 547, 522, 465, 521, 510, 501, 487, 488, 549, 529, 530, 489, 536, 485, 532, 536, 492, 509, 508, 513, 491, 512, 527, 552, 505, 504, 473, 557, 520, 501, 514, 499, 507, 502, 504, 558, 506, 539, 525, 494, 536] payload_rows_per_dst=[5580, 5482, 5644, 5570, 5584, 5552, 5516, 5560] dup_rows_per_dst=[2588, 2562, 2624, 2516, 2674, 2682, 2634, 2768] combine_rows_per_src=[5580, 5482, 5644, 5570, 5584, 5552, 5516, 5560] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=39012 dup_rows=20918 dup_rows_remote=18358 dedup_ratio=0.319183 router_tokens_per_expert=[508, 543, 484, 564, 496, 540, 527, 483, 503, 495, 519, 489, 512, 543, 535, 487, 501, 500, 510, 497, 524, 515, 499, 551, 507, 500, 547, 508, 547, 526, 502, 509, 489, 485, 524, 498, 509, 525, 509, 506, 493, 467, 525, 499, 488, 538, 514, 506, 536, 514, 545, 521, 513, 484, 547, 525, 482, 516, 524, 498, 513, 506, 501, 497] payload_rows_per_dst=[5636, 5606, 5540, 5634, 5458, 5538, 5664, 5542] dup_rows_per_dst=[2654, 2560, 2654, 2658, 2632, 2522, 2706, 2532] combine_rows_per_src=[5636, 5606, 5540, 5634, 5458, 5538, 5664, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38900 dup_rows=21076 dup_rows_remote=18538 dedup_ratio=0.321594 router_tokens_per_expert=[503, 530, 509, 465, 493, 513, 509, 534, 549, 530, 547, 501, 493, 510, 503, 482, 527, 529, 535, 472, 503, 508, 501, 555, 480, 504, 528, 508, 537, 498, 503, 510, 525, 479, 537, 509, 469, 500, 482, 548, 490, 513, 502, 492, 516, 516, 523, 535, 515, 501, 515, 503, 529, 530, 495, 499, 522, 526, 508, 562, 512, 519, 504, 523] payload_rows_per_dst=[5512, 5602, 5562, 5544, 5560, 5556, 5544, 5580] dup_rows_per_dst=[2600, 2628, 2698, 2592, 2538, 2618, 2630, 2772] combine_rows_per_src=[5512, 5602, 5562, 5544, 5560, 5556, 5544, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=229264 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279222160 ingress_bytes=389093376 link_bytes=389093376 token_bytes=558215056 weight_movement_bytes=110100480 total_nvlink_bytes=668315536 weight_to_token_ratio=0.197237 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766973757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279638016 dispatch_meta_bytes=229480 combine_payload_bytes=279638016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279867496 ingress_bytes=389738496 link_bytes=389738496 token_bytes=559505512 weight_movement_bytes=110100480 total_nvlink_bytes=669605992 weight_to_token_ratio=0.196782 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778193455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=229752 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279064952 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557900152 weight_movement_bytes=110100480 total_nvlink_bytes=668000632 weight_to_token_ratio=0.197348 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38896 dup_rows=20992 dup_rows_remote=18370 dedup_ratio=0.320312 router_tokens_per_expert=[502, 530, 531, 514, 537, 527, 513, 508, 540, 500, 518, 505, 537, 516, 499, 502, 488, 507, 542, 543, 508, 529, 522, 491, 495, 477, 532, 453, 525, 511, 470, 526, 520, 506, 504, 498, 534, 522, 524, 503, 497, 494, 478, 535, 527, 505, 498, 503, 530, 551, 492, 511, 501, 521, 536, 493, 563, 500, 517, 519, 490, 508, 515, 475] payload_rows_per_dst=[5602, 5548, 5588, 5470, 5554, 5554, 5648, 5580] dup_rows_per_dst=[2722, 2686, 2672, 2508, 2668, 2520, 2622, 2594] combine_rows_per_src=[5602, 5548, 5588, 5470, 5554, 5554, 5648, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279867392 dispatch_meta_bytes=228928 combine_payload_bytes=279867392 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280096320 ingress_bytes=500068352 link_bytes=500068352 token_bytes=559963712 weight_movement_bytes=220200960 total_nvlink_bytes=780164672 weight_to_token_ratio=0.393241 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.697011112e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279595008 dispatch_meta_bytes=229968 combine_payload_bytes=279595008 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279824976 ingress_bytes=499795968 link_bytes=499795968 token_bytes=559419984 weight_movement_bytes=220200960 total_nvlink_bytes=779620944 weight_to_token_ratio=0.393624 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.692273907e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44732 payload_rows_remote=39138 dup_rows=20804 dup_rows_remote=18214 dedup_ratio=0.317444 router_tokens_per_expert=[484, 526, 519, 536, 539, 499, 522, 496, 527, 514, 511, 482, 515, 478, 507, 504, 532, 506, 538, 492, 534, 527, 531, 546, 531, 520, 518, 524, 500, 483, 515, 499, 509, 504, 488, 525, 515, 505, 543, 529, 543, 511, 507, 463, 516, 519, 531, 502, 522, 488, 500, 500, 532, 516, 512, 492, 512, 511, 491, 506, 512, 488, 539, 482] payload_rows_per_dst=[5614, 5528, 5714, 5552, 5598, 5594, 5556, 5576] dup_rows_per_dst=[2628, 2548, 2698, 2628, 2638, 2590, 2568, 2506] combine_rows_per_src=[5614, 5528, 5714, 5552, 5598, 5594, 5556, 5576] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44626 payload_rows_remote=38958 dup_rows=20910 dup_rows_remote=18262 dedup_ratio=0.319061 router_tokens_per_expert=[523, 486, 465, 509, 505, 535, 489, 519, 503, 512, 483, 530, 490, 525, 510, 498, 521, 523, 505, 502, 536, 484, 498, 503, 499, 513, 523, 544, 500, 509, 568, 531, 520, 489, 509, 494, 508, 510, 548, 512, 547, 562, 524, 525, 514, 463, 538, 510, 517, 507, 520, 500, 493, 481, 522, 499, 527, 501, 517, 495, 511, 510, 539, 515] payload_rows_per_dst=[5352, 5596, 5296, 5756, 5674, 5742, 5542, 5668] dup_rows_per_dst=[2420, 2626, 2244, 2978, 2730, 2708, 2556, 2648] combine_rows_per_src=[5352, 5596, 5296, 5756, 5674, 5742, 5542, 5668] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38830 dup_rows=21122 dup_rows_remote=18534 dedup_ratio=0.322296 router_tokens_per_expert=[504, 529, 497, 515, 522, 505, 509, 549, 511, 542, 508, 511, 535, 518, 518, 547, 515, 521, 493, 508, 524, 516, 524, 485, 503, 522, 506, 505, 527, 524, 474, 481, 512, 531, 509, 504, 475, 556, 477, 520, 505, 507, 494, 482, 499, 464, 503, 513, 503, 515, 530, 552, 505, 520, 565, 473, 479, 510, 480, 487, 535, 539, 514, 562] payload_rows_per_dst=[5580, 5618, 5584, 5440, 5568, 5412, 5630, 5582] dup_rows_per_dst=[2680, 2762, 2588, 2644, 2600, 2522, 2696, 2630] combine_rows_per_src=[5580, 5618, 5584, 5440, 5568, 5412, 5630, 5582] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44576 payload_rows_remote=39098 dup_rows=20960 dup_rows_remote=18418 dedup_ratio=0.319824 router_tokens_per_expert=[511, 455, 499, 520, 536, 551, 508, 528, 502, 495, 534, 496, 512, 487, 513, 528, 538, 545, 534, 500, 510, 527, 551, 512, 508, 529, 507, 461, 515, 517, 475, 498, 489, 490, 513, 542, 483, 481, 507, 533, 520, 515, 530, 488, 509, 523, 562, 513, 509, 505, 500, 522, 526, 482, 513, 525, 518, 493, 480, 532, 548, 473, 540, 502] payload_rows_per_dst=[5536, 5526, 5676, 5478, 5554, 5668, 5586, 5552] dup_rows_per_dst=[2680, 2608, 2758, 2542, 2522, 2652, 2578, 2620] combine_rows_per_src=[5536, 5526, 5676, 5478, 5554, 5668, 5586, 5552] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44544 payload_rows_remote=38896 dup_rows=20992 dup_rows_remote=18370 dedup_ratio=0.320312 router_tokens_per_expert=[502, 530, 531, 514, 537, 527, 513, 508, 540, 500, 518, 505, 537, 516, 499, 502, 488, 507, 542, 543, 508, 529, 522, 491, 495, 477, 532, 453, 525, 511, 470, 526, 520, 506, 504, 498, 534, 522, 524, 503, 497, 494, 478, 535, 527, 505, 498, 503, 530, 551, 492, 511, 501, 521, 536, 493, 563, 500, 517, 519, 490, 508, 515, 475] payload_rows_per_dst=[5602, 5548, 5588, 5470, 5554, 5554, 5648, 5580] dup_rows_per_dst=[2722, 2686, 2672, 2508, 2668, 2520, 2622, 2594] combine_rows_per_src=[5602, 5548, 5588, 5470, 5554, 5554, 5648, 5580] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=229064 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279035592 ingress_bytes=499007488 link_bytes=499007488 token_bytes=557842120 weight_movement_bytes=220200960 total_nvlink_bytes=778043080 weight_to_token_ratio=0.394737 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678560943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38928 dup_rows=21014 dup_rows_remote=18288 dedup_ratio=0.320648 router_tokens_per_expert=[517, 484, 507, 528, 525, 536, 537, 502, 503, 478, 521, 492, 514, 516, 513, 510, 513, 544, 530, 507, 529, 514, 516, 479, 501, 508, 484, 513, 506, 508, 515, 521, 539, 561, 500, 478, 508, 520, 505, 510, 523, 498, 555, 537, 523, 535, 507, 492, 489, 527, 494, 507, 509, 492, 504, 507, 537, 518, 510, 500, 495, 503, 499, 515] payload_rows_per_dst=[5594, 5508, 5544, 5484, 5654, 5596, 5556, 5586] dup_rows_per_dst=[2726, 2586, 2720, 2628, 2588, 2744, 2454, 2568] combine_rows_per_src=[5594, 5508, 5544, 5484, 5654, 5596, 5556, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38900 dup_rows=21076 dup_rows_remote=18538 dedup_ratio=0.321594 router_tokens_per_expert=[503, 530, 509, 465, 493, 513, 509, 534, 549, 530, 547, 501, 493, 510, 503, 482, 527, 529, 535, 472, 503, 508, 501, 555, 480, 504, 528, 508, 537, 498, 503, 510, 525, 479, 537, 509, 469, 500, 482, 548, 490, 513, 502, 492, 516, 516, 523, 535, 515, 501, 515, 503, 529, 530, 495, 499, 522, 526, 508, 562, 512, 519, 504, 523] payload_rows_per_dst=[5512, 5602, 5562, 5544, 5560, 5556, 5544, 5580] dup_rows_per_dst=[2600, 2628, 2698, 2592, 2538, 2618, 2630, 2772] combine_rows_per_src=[5512, 5602, 5562, 5544, 5560, 5556, 5544, 5580] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280541184 dispatch_meta_bytes=229408 combine_payload_bytes=280541184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280770592 ingress_bytes=390641664 link_bytes=390641664 token_bytes=561311776 weight_movement_bytes=110100480 total_nvlink_bytes=671412256 weight_to_token_ratio=0.196149 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.793901032e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=228880 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279479824 ingress_bytes=389351424 link_bytes=389351424 token_bytes=558730768 weight_movement_bytes=110100480 total_nvlink_bytes=668831248 weight_to_token_ratio=0.197055 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771461636e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44564 payload_rows_remote=38978 dup_rows=20972 dup_rows_remote=18326 dedup_ratio=0.320007 router_tokens_per_expert=[476, 480, 522, 547, 491, 518, 537, 531, 499, 532, 526, 499, 550, 467, 497, 546, 559, 518, 530, 514, 540, 524, 516, 534, 543, 527, 498, 486, 492, 483, 530, 547, 475, 501, 560, 530, 493, 481, 485, 494, 460, 503, 482, 530, 514, 517, 515, 494, 516, 479, 493, 520, 543, 551, 543, 532, 479, 508, 499, 530, 494, 508, 497, 483] payload_rows_per_dst=[5672, 5586, 5596, 5550, 5520, 5566, 5640, 5434] dup_rows_per_dst=[2532, 2646, 2874, 2662, 2518, 2464, 2714, 2562] combine_rows_per_src=[5672, 5586, 5596, 5550, 5520, 5566, 5640, 5434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=229456 combine_payload_bytes=278333440 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278562896 ingress_bytes=608634880 link_bytes=608634880 token_bytes=556896336 weight_movement_bytes=330301440 total_nvlink_bytes=887197776 weight_to_token_ratio=0.593111 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058516160e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280254464 dispatch_meta_bytes=230064 combine_payload_bytes=280254464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280484528 ingress_bytes=390354944 link_bytes=390354944 token_bytes=560738992 weight_movement_bytes=110100480 total_nvlink_bytes=670839472 weight_to_token_ratio=0.196349 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788914500e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44236 payload_rows_remote=38660 dup_rows=21300 dup_rows_remote=18596 dedup_ratio=0.325012 router_tokens_per_expert=[532, 493, 529, 498, 529, 494, 489, 495, 509, 532, 534, 480, 508, 514, 541, 520, 484, 534, 518, 505, 501, 509, 493, 497, 529, 526, 495, 511, 483, 494, 488, 514, 548, 502, 565, 510, 504, 492, 515, 504, 481, 501, 520, 520, 483, 511, 541, 527, 517, 507, 526, 536, 528, 537, 534, 481, 517, 512, 485, 518, 519, 523, 516, 510] payload_rows_per_dst=[5450, 5580, 5488, 5490, 5576, 5466, 5588, 5598] dup_rows_per_dst=[2668, 2696, 2594, 2590, 2704, 2702, 2744, 2602] combine_rows_per_src=[5450, 5580, 5488, 5490, 5576, 5466, 5588, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38928 dup_rows=21014 dup_rows_remote=18288 dedup_ratio=0.320648 router_tokens_per_expert=[517, 484, 507, 528, 525, 536, 537, 502, 503, 478, 521, 492, 514, 516, 513, 510, 513, 544, 530, 507, 529, 514, 516, 479, 501, 508, 484, 513, 506, 508, 515, 521, 539, 561, 500, 478, 508, 520, 505, 510, 523, 498, 555, 537, 523, 535, 507, 492, 489, 527, 494, 507, 509, 492, 504, 507, 537, 518, 510, 500, 495, 503, 499, 515] payload_rows_per_dst=[5594, 5508, 5544, 5484, 5654, 5596, 5556, 5586] dup_rows_per_dst=[2726, 2586, 2720, 2628, 2588, 2744, 2454, 2568] combine_rows_per_src=[5594, 5508, 5544, 5484, 5654, 5596, 5556, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=229064 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279035592 ingress_bytes=499007488 link_bytes=499007488 token_bytes=557842120 weight_movement_bytes=220200960 total_nvlink_bytes=778043080 weight_to_token_ratio=0.394737 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678560943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278835200 dispatch_meta_bytes=229752 combine_payload_bytes=278835200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279064952 ingress_bytes=388935680 link_bytes=388935680 token_bytes=557900152 weight_movement_bytes=110100480 total_nvlink_bytes=668000632 weight_to_token_ratio=0.197348 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764231164e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=39012 dup_rows=20918 dup_rows_remote=18358 dedup_ratio=0.319183 router_tokens_per_expert=[508, 543, 484, 564, 496, 540, 527, 483, 503, 495, 519, 489, 512, 543, 535, 487, 501, 500, 510, 497, 524, 515, 499, 551, 507, 500, 547, 508, 547, 526, 502, 509, 489, 485, 524, 498, 509, 525, 509, 506, 493, 467, 525, 499, 488, 538, 514, 506, 536, 514, 545, 521, 513, 484, 547, 525, 482, 516, 524, 498, 513, 506, 501, 497] payload_rows_per_dst=[5636, 5606, 5540, 5634, 5458, 5538, 5664, 5542] dup_rows_per_dst=[2654, 2560, 2654, 2658, 2632, 2522, 2706, 2532] combine_rows_per_src=[5636, 5606, 5540, 5634, 5458, 5538, 5664, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38830 dup_rows=21122 dup_rows_remote=18534 dedup_ratio=0.322296 router_tokens_per_expert=[504, 529, 497, 515, 522, 505, 509, 549, 511, 542, 508, 511, 535, 518, 518, 547, 515, 521, 493, 508, 524, 516, 524, 485, 503, 522, 506, 505, 527, 524, 474, 481, 512, 531, 509, 504, 475, 556, 477, 520, 505, 507, 494, 482, 499, 464, 503, 513, 503, 515, 530, 552, 505, 520, 565, 473, 479, 510, 480, 487, 535, 539, 514, 562] payload_rows_per_dst=[5580, 5618, 5584, 5440, 5568, 5412, 5630, 5582] dup_rows_per_dst=[2680, 2762, 2588, 2644, 2600, 2522, 2696, 2630] combine_rows_per_src=[5580, 5618, 5584, 5440, 5568, 5412, 5630, 5582] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279035904 dispatch_meta_bytes=228864 combine_payload_bytes=279035904 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279264768 ingress_bytes=719437824 link_bytes=719437824 token_bytes=558300672 weight_movement_bytes=440401920 total_nvlink_bytes=998702592 weight_to_token_ratio=0.788826 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251220703e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=229216 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279623520 ingress_bytes=499595264 link_bytes=499595264 token_bytes=559017824 weight_movement_bytes=220200960 total_nvlink_bytes=779218784 weight_to_token_ratio=0.393907 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688783334e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277114880 dispatch_meta_bytes=229024 combine_payload_bytes=277114880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277343904 ingress_bytes=387215360 link_bytes=387215360 token_bytes=554458784 weight_movement_bytes=110100480 total_nvlink_bytes=664559264 weight_to_token_ratio=0.198573 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.734311970e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38864 dup_rows=21114 dup_rows_remote=18404 dedup_ratio=0.322174 router_tokens_per_expert=[479, 476, 515, 534, 516, 502, 517, 479, 498, 520, 505, 518, 472, 548, 500, 519, 516, 498, 527, 506, 512, 494, 506, 569, 534, 501, 513, 535, 496, 496, 506, 504, 522, 505, 484, 525, 480, 504, 545, 532, 510, 487, 525, 532, 541, 456, 512, 527, 497, 513, 479, 510, 527, 553, 539, 516, 539, 537, 517, 484, 526, 492, 514, 527] payload_rows_per_dst=[5566, 5562, 5532, 5504, 5558, 5560, 5558, 5582] dup_rows_per_dst=[2470, 2598, 2724, 2666, 2636, 2620, 2710, 2690] combine_rows_per_src=[5566, 5562, 5532, 5504, 5558, 5560, 5558, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38766 dup_rows=21082 dup_rows_remote=18318 dedup_ratio=0.321686 router_tokens_per_expert=[512, 476, 519, 511, 478, 527, 494, 508, 533, 509, 477, 492, 533, 513, 531, 508, 498, 508, 524, 499, 513, 492, 503, 524, 536, 499, 517, 505, 499, 510, 497, 517, 524, 512, 510, 518, 542, 504, 489, 491, 566, 521, 500, 556, 532, 525, 524, 502, 496, 527, 490, 525, 531, 466, 537, 536, 518, 478, 509, 488, 534, 501, 515, 539] payload_rows_per_dst=[5528, 5498, 5566, 5516, 5594, 5688, 5520, 5544] dup_rows_per_dst=[2522, 2694, 2556, 2644, 2586, 2764, 2696, 2620] combine_rows_per_src=[5528, 5498, 5566, 5516, 5594, 5688, 5520, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279638016 dispatch_meta_bytes=229480 combine_payload_bytes=279638016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279867496 ingress_bytes=389738496 link_bytes=389738496 token_bytes=559505512 weight_movement_bytes=110100480 total_nvlink_bytes=669605992 weight_to_token_ratio=0.196782 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778193455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278333440 dispatch_meta_bytes=229456 combine_payload_bytes=278333440 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278562896 ingress_bytes=608634880 link_bytes=608634880 token_bytes=556896336 weight_movement_bytes=330301440 total_nvlink_bytes=887197776 weight_to_token_ratio=0.593111 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058516160e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38886 dup_rows=21064 dup_rows_remote=18368 dedup_ratio=0.321411 router_tokens_per_expert=[469, 477, 503, 550, 532, 511, 506, 532, 522, 479, 499, 498, 541, 523, 529, 498, 530, 471, 531, 521, 493, 541, 523, 506, 524, 516, 500, 533, 523, 534, 519, 491, 495, 539, 504, 513, 541, 460, 494, 476, 521, 496, 559, 480, 517, 512, 522, 475, 491, 508, 530, 502, 500, 550, 522, 512, 501, 496, 508, 492, 519, 558, 522, 528] payload_rows_per_dst=[5586, 5544, 5524, 5630, 5482, 5506, 5614, 5586] dup_rows_per_dst=[2574, 2614, 2708, 2636, 2562, 2532, 2742, 2696] combine_rows_per_src=[5586, 5544, 5524, 5630, 5482, 5506, 5614, 5586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38852 dup_rows=21070 dup_rows_remote=18420 dedup_ratio=0.321503 router_tokens_per_expert=[508, 506, 489, 521, 525, 526, 541, 535, 516, 530, 509, 506, 529, 541, 532, 455, 520, 512, 513, 521, 502, 548, 511, 505, 506, 527, 499, 538, 536, 513, 535, 501, 508, 496, 489, 496, 524, 462, 496, 508, 509, 526, 513, 528, 496, 507, 509, 490, 497, 482, 484, 509, 527, 510, 531, 504, 516, 487, 515, 524, 510, 542, 483, 534] payload_rows_per_dst=[5646, 5540, 5614, 5592, 5436, 5536, 5552, 5550] dup_rows_per_dst=[2656, 2696, 2650, 2718, 2522, 2620, 2536, 2672] combine_rows_per_src=[5646, 5540, 5614, 5592, 5436, 5536, 5552, 5550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44766 payload_rows_remote=39122 dup_rows=20770 dup_rows_remote=18208 dedup_ratio=0.316925 router_tokens_per_expert=[500, 489, 506, 527, 500, 492, 538, 533, 512, 507, 483, 494, 501, 473, 512, 498, 541, 465, 500, 527, 510, 513, 519, 508, 491, 519, 518, 490, 510, 543, 491, 541, 514, 486, 497, 515, 526, 529, 552, 515, 484, 539, 498, 545, 530, 538, 535, 527, 497, 502, 546, 512, 500, 523, 537, 483, 513, 523, 516, 498, 508, 491, 529, 509] payload_rows_per_dst=[5618, 5522, 5512, 5644, 5630, 5620, 5594, 5626] dup_rows_per_dst=[2552, 2438, 2654, 2562, 2638, 2772, 2606, 2548] combine_rows_per_src=[5618, 5522, 5512, 5644, 5630, 5620, 5594, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229072 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278806224 ingress_bytes=498778112 link_bytes=498778112 token_bytes=557383376 weight_movement_bytes=220200960 total_nvlink_bytes=777584336 weight_to_token_ratio=0.395062 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.674571717e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277874688 dispatch_meta_bytes=228336 combine_payload_bytes=277874688 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278103024 ingress_bytes=498075648 link_bytes=498075648 token_bytes=555977712 weight_movement_bytes=220200960 total_nvlink_bytes=776178672 weight_to_token_ratio=0.396061 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662354712e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38864 dup_rows=21114 dup_rows_remote=18404 dedup_ratio=0.322174 router_tokens_per_expert=[479, 476, 515, 534, 516, 502, 517, 479, 498, 520, 505, 518, 472, 548, 500, 519, 516, 498, 527, 506, 512, 494, 506, 569, 534, 501, 513, 535, 496, 496, 506, 504, 522, 505, 484, 525, 480, 504, 545, 532, 510, 487, 525, 532, 541, 456, 512, 527, 497, 513, 479, 510, 527, 553, 539, 516, 539, 537, 517, 484, 526, 492, 514, 527] payload_rows_per_dst=[5566, 5562, 5532, 5504, 5558, 5560, 5558, 5582] dup_rows_per_dst=[2470, 2598, 2724, 2666, 2636, 2620, 2710, 2690] combine_rows_per_src=[5566, 5562, 5532, 5504, 5558, 5560, 5558, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44236 payload_rows_remote=38660 dup_rows=21300 dup_rows_remote=18596 dedup_ratio=0.325012 router_tokens_per_expert=[532, 493, 529, 498, 529, 494, 489, 495, 509, 532, 534, 480, 508, 514, 541, 520, 484, 534, 518, 505, 501, 509, 493, 497, 529, 526, 495, 511, 483, 494, 488, 514, 548, 502, 565, 510, 504, 492, 515, 504, 481, 501, 520, 520, 483, 511, 541, 527, 517, 507, 526, 536, 528, 537, 534, 481, 517, 512, 485, 518, 519, 523, 516, 510] payload_rows_per_dst=[5450, 5580, 5488, 5490, 5576, 5466, 5588, 5598] dup_rows_per_dst=[2668, 2696, 2594, 2590, 2704, 2702, 2744, 2602] combine_rows_per_src=[5450, 5580, 5488, 5490, 5576, 5466, 5588, 5598] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44542 payload_rows_remote=39004 dup_rows=20994 dup_rows_remote=18470 dedup_ratio=0.320343 router_tokens_per_expert=[485, 484, 507, 538, 545, 505, 486, 503, 523, 458, 519, 488, 493, 500, 542, 508, 477, 505, 463, 498, 499, 502, 503, 534, 503, 551, 544, 521, 520, 492, 505, 565, 513, 534, 531, 520, 522, 518, 505, 538, 526, 483, 492, 491, 503, 503, 543, 508, 556, 489, 532, 495, 504, 492, 517, 548, 481, 523, 498, 555, 508, 524, 531, 519] payload_rows_per_dst=[5504, 5538, 5536, 5686, 5610, 5528, 5570, 5570] dup_rows_per_dst=[2602, 2524, 2426, 2716, 2752, 2570, 2696, 2708] combine_rows_per_src=[5504, 5538, 5536, 5686, 5610, 5528, 5570, 5570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38722 dup_rows=21134 dup_rows_remote=18318 dedup_ratio=0.322479 router_tokens_per_expert=[487, 508, 512, 525, 533, 512, 504, 505, 522, 509, 542, 524, 512, 514, 503, 530, 524, 518, 510, 515, 493, 509, 535, 485, 529, 511, 502, 513, 501, 528, 527, 500, 531, 508, 523, 501, 490, 521, 548, 513, 494, 500, 517, 498, 521, 535, 509, 491, 492, 492, 499, 509, 516, 495, 519, 491, 514, 528, 525, 487, 518, 499, 539, 503] payload_rows_per_dst=[5680, 5588, 5470, 5520, 5680, 5508, 5398, 5558] dup_rows_per_dst=[2816, 2734, 2474, 2612, 2894, 2612, 2324, 2668] combine_rows_per_src=[5680, 5588, 5470, 5520, 5680, 5508, 5398, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277874688 dispatch_meta_bytes=228336 combine_payload_bytes=277874688 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278103024 ingress_bytes=498075648 link_bytes=498075648 token_bytes=555977712 weight_movement_bytes=220200960 total_nvlink_bytes=776178672 weight_to_token_ratio=0.396061 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662354712e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=229016 combine_payload_bytes=278734848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278963864 ingress_bytes=388835328 link_bytes=388835328 token_bytes=557698712 weight_movement_bytes=110100480 total_nvlink_bytes=667799192 weight_to_token_ratio=0.197419 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762485878e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38946 dup_rows=21082 dup_rows_remote=18484 dedup_ratio=0.321686 router_tokens_per_expert=[487, 512, 512, 510, 513, 509, 511, 520, 535, 519, 521, 565, 520, 494, 504, 534, 487, 519, 510, 526, 520, 503, 467, 497, 520, 514, 498, 489, 527, 523, 501, 525, 493, 494, 495, 538, 487, 494, 524, 528, 530, 524, 533, 507, 500, 491, 465, 557, 520, 473, 549, 518, 527, 526, 536, 522, 493, 515, 544, 506, 496, 481, 492, 518] payload_rows_per_dst=[5570, 5616, 5550, 5524, 5508, 5558, 5594, 5534] dup_rows_per_dst=[2578, 2768, 2508, 2670, 2598, 2656, 2748, 2556] combine_rows_per_src=[5570, 5616, 5550, 5524, 5508, 5558, 5594, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44402 payload_rows_remote=38722 dup_rows=21134 dup_rows_remote=18318 dedup_ratio=0.322479 router_tokens_per_expert=[487, 508, 512, 525, 533, 512, 504, 505, 522, 509, 542, 524, 512, 514, 503, 530, 524, 518, 510, 515, 493, 509, 535, 485, 529, 511, 502, 513, 501, 528, 527, 500, 531, 508, 523, 501, 490, 521, 548, 513, 494, 500, 517, 498, 521, 535, 509, 491, 492, 492, 499, 509, 516, 495, 519, 491, 514, 528, 525, 487, 518, 499, 539, 503] payload_rows_per_dst=[5680, 5588, 5470, 5520, 5680, 5508, 5398, 5558] dup_rows_per_dst=[2816, 2734, 2474, 2612, 2894, 2612, 2324, 2668] combine_rows_per_src=[5680, 5588, 5470, 5520, 5680, 5508, 5398, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280426496 dispatch_meta_bytes=229320 combine_payload_bytes=280426496 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280655816 ingress_bytes=610727936 link_bytes=610727936 token_bytes=561082312 weight_movement_bytes=330301440 total_nvlink_bytes=891383752 weight_to_token_ratio=0.588686 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062156328e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=229088 combine_payload_bytes=278491136 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278720224 ingress_bytes=498692096 link_bytes=498692096 token_bytes=557211360 weight_movement_bytes=220200960 total_nvlink_bytes=777412320 weight_to_token_ratio=0.395184 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673075757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279580672 dispatch_meta_bytes=229896 combine_payload_bytes=279580672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279810568 ingress_bytes=389681152 link_bytes=389681152 token_bytes=559391240 weight_movement_bytes=110100480 total_nvlink_bytes=669491720 weight_to_token_ratio=0.196822 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777196149e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229720 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279394648 ingress_bytes=499365888 link_bytes=499365888 token_bytes=558559576 weight_movement_bytes=220200960 total_nvlink_bytes=778760536 weight_to_token_ratio=0.394230 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684794108e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229072 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278806224 ingress_bytes=498778112 link_bytes=498778112 token_bytes=557383376 weight_movement_bytes=220200960 total_nvlink_bytes=777584336 weight_to_token_ratio=0.395062 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.674571717e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277114880 dispatch_meta_bytes=229024 combine_payload_bytes=277114880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277343904 ingress_bytes=387215360 link_bytes=387215360 token_bytes=554458784 weight_movement_bytes=110100480 total_nvlink_bytes=664559264 weight_to_token_ratio=0.198573 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.734311970e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44564 payload_rows_remote=38978 dup_rows=20972 dup_rows_remote=18326 dedup_ratio=0.320007 router_tokens_per_expert=[476, 480, 522, 547, 491, 518, 537, 531, 499, 532, 526, 499, 550, 467, 497, 546, 559, 518, 530, 514, 540, 524, 516, 534, 543, 527, 498, 486, 492, 483, 530, 547, 475, 501, 560, 530, 493, 481, 485, 494, 460, 503, 482, 530, 514, 517, 515, 494, 516, 479, 493, 520, 543, 551, 543, 532, 479, 508, 499, 530, 494, 508, 497, 483] payload_rows_per_dst=[5672, 5586, 5596, 5550, 5520, 5566, 5640, 5434] dup_rows_per_dst=[2532, 2646, 2874, 2662, 2518, 2464, 2714, 2562] combine_rows_per_src=[5672, 5586, 5596, 5550, 5520, 5566, 5640, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44466 payload_rows_remote=38852 dup_rows=21070 dup_rows_remote=18420 dedup_ratio=0.321503 router_tokens_per_expert=[508, 506, 489, 521, 525, 526, 541, 535, 516, 530, 509, 506, 529, 541, 532, 455, 520, 512, 513, 521, 502, 548, 511, 505, 506, 527, 499, 538, 536, 513, 535, 501, 508, 496, 489, 496, 524, 462, 496, 508, 509, 526, 513, 528, 496, 507, 509, 490, 497, 482, 484, 509, 527, 510, 531, 504, 516, 487, 515, 524, 510, 542, 483, 534] payload_rows_per_dst=[5646, 5540, 5614, 5592, 5436, 5536, 5552, 5550] dup_rows_per_dst=[2656, 2696, 2650, 2718, 2522, 2620, 2536, 2672] combine_rows_per_src=[5646, 5540, 5614, 5592, 5436, 5536, 5552, 5550] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277559296 dispatch_meta_bytes=228160 combine_payload_bytes=277559296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277787456 ingress_bytes=387659776 link_bytes=387659776 token_bytes=555346752 weight_movement_bytes=110100480 total_nvlink_bytes=665447232 weight_to_token_ratio=0.198255 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.742041095e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44508 payload_rows_remote=38948 dup_rows=21028 dup_rows_remote=18410 dedup_ratio=0.320862 router_tokens_per_expert=[529, 516, 520, 495, 512, 527, 525, 478, 496, 482, 508, 484, 470, 536, 510, 495, 491, 465, 510, 500, 535, 512, 554, 474, 549, 504, 551, 532, 536, 572, 521, 514, 514, 520, 510, 521, 504, 509, 538, 512, 506, 496, 522, 507, 536, 486, 484, 536, 570, 516, 510, 508, 482, 511, 530, 462, 479, 520, 504, 544, 524, 489, 513, 502] payload_rows_per_dst=[5532, 5506, 5480, 5734, 5570, 5552, 5560, 5574] dup_rows_per_dst=[2672, 2456, 2602, 2824, 2686, 2594, 2618, 2576] combine_rows_per_src=[5532, 5506, 5480, 5734, 5570, 5552, 5560, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44628 payload_rows_remote=39012 dup_rows=20908 dup_rows_remote=18280 dedup_ratio=0.319031 router_tokens_per_expert=[474, 532, 493, 479, 498, 487, 507, 504, 515, 563, 510, 536, 510, 525, 455, 477, 542, 541, 470, 508, 502, 511, 544, 532, 534, 510, 540, 496, 493, 524, 514, 496, 499, 530, 490, 504, 532, 525, 538, 534, 530, 523, 532, 528, 511, 493, 485, 520, 510, 516, 516, 527, 531, 529, 476, 498, 492, 494, 489, 516, 490, 530, 536, 522] payload_rows_per_dst=[5460, 5594, 5602, 5632, 5610, 5616, 5586, 5528] dup_rows_per_dst=[2488, 2588, 2698, 2582, 2694, 2628, 2620, 2610] combine_rows_per_src=[5460, 5594, 5602, 5632, 5610, 5616, 5586, 5528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38752 dup_rows=21108 dup_rows_remote=18238 dedup_ratio=0.322083 router_tokens_per_expert=[512, 513, 572, 513, 529, 524, 514, 516, 487, 522, 510, 508, 537, 529, 487, 501, 514, 540, 526, 508, 525, 484, 536, 543, 485, 500, 513, 505, 516, 511, 499, 547, 524, 485, 496, 504, 484, 508, 515, 505, 537, 505, 484, 532, 509, 519, 501, 550, 499, 522, 520, 488, 468, 521, 498, 495, 540, 522, 490, 496, 486, 528, 495, 516] payload_rows_per_dst=[5588, 5534, 5652, 5418, 5478, 5612, 5470, 5676] dup_rows_per_dst=[2750, 2628, 2700, 2360, 2494, 2754, 2552, 2870] combine_rows_per_src=[5588, 5534, 5652, 5418, 5478, 5612, 5470, 5676] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279179264 dispatch_meta_bytes=229432 combine_payload_bytes=279179264 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279408696 ingress_bytes=389279744 link_bytes=389279744 token_bytes=558587960 weight_movement_bytes=110100480 total_nvlink_bytes=668688440 weight_to_token_ratio=0.197105 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770215003e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=229216 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279623520 ingress_bytes=499595264 link_bytes=499595264 token_bytes=559017824 weight_movement_bytes=220200960 total_nvlink_bytes=779218784 weight_to_token_ratio=0.393907 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688783334e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=229088 combine_payload_bytes=278491136 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278720224 ingress_bytes=498692096 link_bytes=498692096 token_bytes=557211360 weight_movement_bytes=220200960 total_nvlink_bytes=777412320 weight_to_token_ratio=0.395184 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.673075757e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44652 payload_rows_remote=39074 dup_rows=20884 dup_rows_remote=18258 dedup_ratio=0.318665 router_tokens_per_expert=[511, 499, 515, 528, 538, 507, 518, 504, 515, 539, 526, 456, 539, 522, 536, 498, 500, 490, 502, 477, 535, 490, 538, 500, 505, 482, 511, 530, 494, 551, 539, 490, 537, 544, 509, 505, 494, 467, 508, 522, 519, 506, 539, 507, 570, 533, 501, 508, 522, 525, 472, 497, 491, 476, 542, 515, 504, 488, 518, 497, 525, 533, 516, 493] payload_rows_per_dst=[5532, 5616, 5554, 5578, 5630, 5646, 5512, 5584] dup_rows_per_dst=[2708, 2646, 2510, 2626, 2542, 2720, 2568, 2564] combine_rows_per_src=[5532, 5616, 5554, 5578, 5630, 5646, 5512, 5584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44276 payload_rows_remote=38804 dup_rows=21260 dup_rows_remote=18564 dedup_ratio=0.324402 router_tokens_per_expert=[532, 535, 532, 494, 512, 541, 464, 516, 502, 509, 481, 496, 525, 526, 510, 507, 520, 493, 516, 505, 518, 489, 521, 522, 515, 545, 530, 525, 563, 511, 492, 549, 508, 525, 512, 519, 516, 485, 511, 526, 482, 518, 510, 556, 520, 509, 489, 469, 509, 547, 531, 465, 533, 510, 499, 513, 486, 502, 502, 489, 529, 505, 493, 504] payload_rows_per_dst=[5560, 5434, 5472, 5662, 5580, 5532, 5552, 5484] dup_rows_per_dst=[2692, 2678, 2696, 2798, 2624, 2574, 2662, 2536] combine_rows_per_src=[5560, 5434, 5472, 5662, 5580, 5532, 5552, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279638016 dispatch_meta_bytes=229168 combine_payload_bytes=279638016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279867184 ingress_bytes=499838976 link_bytes=499838976 token_bytes=559505200 weight_movement_bytes=220200960 total_nvlink_bytes=779706160 weight_to_token_ratio=0.393564 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693021886e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38820 dup_rows=21094 dup_rows_remote=18436 dedup_ratio=0.321869 router_tokens_per_expert=[507, 472, 518, 530, 526, 493, 521, 505, 517, 561, 556, 509, 488, 517, 492, 500, 515, 513, 514, 521, 528, 480, 503, 535, 469, 536, 526, 487, 512, 532, 493, 495, 511, 518, 530, 538, 457, 507, 522, 512, 534, 499, 500, 497, 513, 504, 518, 526, 509, 518, 501, 499, 514, 486, 533, 504, 529, 523, 519, 539, 515, 513, 497, 512] payload_rows_per_dst=[5520, 5622, 5494, 5548, 5524, 5552, 5556, 5626] dup_rows_per_dst=[2624, 2658, 2724, 2552, 2666, 2630, 2572, 2668] combine_rows_per_src=[5520, 5622, 5494, 5548, 5524, 5552, 5556, 5626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44614 payload_rows_remote=39016 dup_rows=20922 dup_rows_remote=18332 dedup_ratio=0.319244 router_tokens_per_expert=[511, 498, 492, 519, 495, 546, 545, 552, 498, 514, 551, 526, 511, 514, 542, 508, 480, 492, 507, 503, 547, 503, 544, 514, 477, 491, 502, 526, 470, 506, 483, 512, 498, 509, 526, 524, 515, 510, 500, 512, 510, 559, 452, 564, 509, 526, 497, 507, 498, 545, 513, 518, 490, 521, 519, 527, 522, 538, 494, 499, 505, 497, 492, 493] payload_rows_per_dst=[5714, 5662, 5538, 5472, 5598, 5492, 5592, 5546] dup_rows_per_dst=[2602, 2666, 2642, 2462, 2590, 2756, 2670, 2534] combine_rows_per_src=[5714, 5662, 5538, 5472, 5598, 5492, 5592, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44508 payload_rows_remote=38948 dup_rows=21028 dup_rows_remote=18410 dedup_ratio=0.320862 router_tokens_per_expert=[529, 516, 520, 495, 512, 527, 525, 478, 496, 482, 508, 484, 470, 536, 510, 495, 491, 465, 510, 500, 535, 512, 554, 474, 549, 504, 551, 532, 536, 572, 521, 514, 514, 520, 510, 521, 504, 509, 538, 512, 506, 496, 522, 507, 536, 486, 484, 536, 570, 516, 510, 508, 482, 511, 530, 462, 479, 520, 504, 544, 524, 489, 513, 502] payload_rows_per_dst=[5532, 5506, 5480, 5734, 5570, 5552, 5560, 5574] dup_rows_per_dst=[2672, 2456, 2602, 2824, 2686, 2594, 2618, 2576] combine_rows_per_src=[5532, 5506, 5480, 5734, 5570, 5552, 5560, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44454 payload_rows_remote=38946 dup_rows=21082 dup_rows_remote=18484 dedup_ratio=0.321686 router_tokens_per_expert=[487, 512, 512, 510, 513, 509, 511, 520, 535, 519, 521, 565, 520, 494, 504, 534, 487, 519, 510, 526, 520, 503, 467, 497, 520, 514, 498, 489, 527, 523, 501, 525, 493, 494, 495, 538, 487, 494, 524, 528, 530, 524, 533, 507, 500, 491, 465, 557, 520, 473, 549, 518, 527, 526, 536, 522, 493, 515, 544, 506, 496, 481, 492, 518] payload_rows_per_dst=[5570, 5616, 5550, 5524, 5508, 5558, 5594, 5534] dup_rows_per_dst=[2578, 2768, 2508, 2670, 2598, 2656, 2748, 2556] combine_rows_per_src=[5570, 5616, 5550, 5524, 5508, 5558, 5594, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279638016 dispatch_meta_bytes=229168 combine_payload_bytes=279638016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279867184 ingress_bytes=499838976 link_bytes=499838976 token_bytes=559505200 weight_movement_bytes=220200960 total_nvlink_bytes=779706160 weight_to_token_ratio=0.393564 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693021886e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=277774336 dispatch_meta_bytes=227960 combine_payload_bytes=277774336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278002296 ingress_bytes=387874816 link_bytes=387874816 token_bytes=555776632 weight_movement_bytes=110100480 total_nvlink_bytes=665877112 weight_to_token_ratio=0.198102 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.745780995e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38916 dup_rows=21042 dup_rows_remote=18320 dedup_ratio=0.321075 router_tokens_per_expert=[498, 566, 493, 515, 503, 527, 527, 520, 550, 490, 531, 532, 501, 506, 510, 499, 490, 547, 498, 474, 494, 524, 514, 506, 497, 491, 516, 573, 514, 519, 499, 482, 534, 550, 524, 503, 464, 515, 475, 515, 533, 518, 490, 501, 500, 524, 536, 482, 524, 511, 519, 508, 464, 564, 515, 510, 500, 511, 501, 513, 510, 531, 504, 513] payload_rows_per_dst=[5578, 5614, 5706, 5426, 5594, 5502, 5620, 5454] dup_rows_per_dst=[2722, 2796, 2844, 2300, 2566, 2494, 2608, 2712] combine_rows_per_src=[5578, 5614, 5706, 5426, 5594, 5502, 5620, 5454] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280082432 dispatch_meta_bytes=229328 combine_payload_bytes=280082432 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280311760 ingress_bytes=610383872 link_bytes=610383872 token_bytes=560394192 weight_movement_bytes=330301440 total_nvlink_bytes=890695632 weight_to_token_ratio=0.589409 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061557944e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=229472 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278376544 ingress_bytes=388247552 link_bytes=388247552 token_bytes=556523616 weight_movement_bytes=110100480 total_nvlink_bytes=666624096 weight_to_token_ratio=0.197836 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752263487e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=38916 dup_rows=21042 dup_rows_remote=18320 dedup_ratio=0.321075 router_tokens_per_expert=[498, 566, 493, 515, 503, 527, 527, 520, 550, 490, 531, 532, 501, 506, 510, 499, 490, 547, 498, 474, 494, 524, 514, 506, 497, 491, 516, 573, 514, 519, 499, 482, 534, 550, 524, 503, 464, 515, 475, 515, 533, 518, 490, 501, 500, 524, 536, 482, 524, 511, 519, 508, 464, 564, 515, 510, 500, 511, 501, 513, 510, 531, 504, 513] payload_rows_per_dst=[5578, 5614, 5706, 5426, 5594, 5502, 5620, 5454] dup_rows_per_dst=[2722, 2796, 2844, 2300, 2566, 2494, 2608, 2712] combine_rows_per_src=[5578, 5614, 5706, 5426, 5594, 5502, 5620, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279179264 dispatch_meta_bytes=229432 combine_payload_bytes=279179264 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=279408696 ingress_bytes=389279744 link_bytes=389279744 token_bytes=558587960 weight_movement_bytes=110100480 total_nvlink_bytes=668688440 weight_to_token_ratio=0.197105 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770215003e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=229024 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278490784 ingress_bytes=498462720 link_bytes=498462720 token_bytes=556752544 weight_movement_bytes=220200960 total_nvlink_bytes=776953504 weight_to_token_ratio=0.395510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669086531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=229392 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279896080 ingress_bytes=499867648 link_bytes=499867648 token_bytes=559562768 weight_movement_bytes=220200960 total_nvlink_bytes=779763728 weight_to_token_ratio=0.393523 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693520540e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44542 payload_rows_remote=39004 dup_rows=20994 dup_rows_remote=18470 dedup_ratio=0.320343 router_tokens_per_expert=[485, 484, 507, 538, 545, 505, 486, 503, 523, 458, 519, 488, 493, 500, 542, 508, 477, 505, 463, 498, 499, 502, 503, 534, 503, 551, 544, 521, 520, 492, 505, 565, 513, 534, 531, 520, 522, 518, 505, 538, 526, 483, 492, 491, 503, 503, 543, 508, 556, 489, 532, 495, 504, 492, 517, 548, 481, 523, 498, 555, 508, 524, 531, 519] payload_rows_per_dst=[5504, 5538, 5536, 5686, 5610, 5528, 5570, 5570] dup_rows_per_dst=[2602, 2524, 2426, 2716, 2752, 2570, 2696, 2708] combine_rows_per_src=[5504, 5538, 5536, 5686, 5610, 5528, 5570, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229720 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279394648 ingress_bytes=499365888 link_bytes=499365888 token_bytes=558559576 weight_movement_bytes=220200960 total_nvlink_bytes=778760536 weight_to_token_ratio=0.394230 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.684794108e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=228944 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279178832 ingress_bytes=609251328 link_bytes=609251328 token_bytes=558128720 weight_movement_bytes=330301440 total_nvlink_bytes=888430160 weight_to_token_ratio=0.591802 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059588264e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38884 dup_rows=21100 dup_rows_remote=18396 dedup_ratio=0.321960 router_tokens_per_expert=[469, 500, 514, 560, 483, 511, 497, 534, 523, 517, 491, 532, 518, 503, 508, 513, 525, 506, 518, 497, 480, 518, 508, 550, 499, 541, 523, 561, 510, 546, 475, 495, 470, 531, 500, 507, 488, 501, 480, 521, 519, 518, 459, 502, 501, 522, 510, 550, 513, 523, 528, 492, 489, 507, 495, 581, 509, 519, 515, 513, 487, 519, 566, 508] payload_rows_per_dst=[5514, 5624, 5534, 5578, 5428, 5582, 5552, 5624] dup_rows_per_dst=[2622, 2586, 2670, 2722, 2568, 2580, 2704, 2648] combine_rows_per_src=[5514, 5624, 5534, 5578, 5428, 5582, 5552, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44276 payload_rows_remote=38804 dup_rows=21260 dup_rows_remote=18564 dedup_ratio=0.324402 router_tokens_per_expert=[532, 535, 532, 494, 512, 541, 464, 516, 502, 509, 481, 496, 525, 526, 510, 507, 520, 493, 516, 505, 518, 489, 521, 522, 515, 545, 530, 525, 563, 511, 492, 549, 508, 525, 512, 519, 516, 485, 511, 526, 482, 518, 510, 556, 520, 509, 489, 469, 509, 547, 531, 465, 533, 510, 499, 513, 486, 502, 502, 489, 529, 505, 493, 504] payload_rows_per_dst=[5560, 5434, 5472, 5662, 5580, 5532, 5552, 5484] dup_rows_per_dst=[2692, 2678, 2696, 2798, 2624, 2574, 2662, 2536] combine_rows_per_src=[5560, 5434, 5472, 5662, 5580, 5532, 5552, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38920 dup_rows=21168 dup_rows_remote=18536 dedup_ratio=0.322998 router_tokens_per_expert=[537, 494, 570, 495, 534, 512, 546, 532, 526, 530, 485, 495, 514, 516, 481, 476, 494, 541, 467, 506, 524, 534, 513, 507, 513, 491, 533, 465, 538, 483, 534, 530, 536, 519, 524, 525, 509, 549, 532, 510, 513, 492, 500, 488, 515, 517, 535, 480, 488, 532, 507, 499, 515, 529, 512, 506, 505, 480, 482, 514, 507, 485, 533, 514] payload_rows_per_dst=[5610, 5516, 5564, 5494, 5694, 5448, 5550, 5492] dup_rows_per_dst=[2830, 2530, 2608, 2680, 2714, 2632, 2626, 2548] combine_rows_per_src=[5610, 5516, 5564, 5494, 5694, 5448, 5550, 5492] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588889760 prefetch_bytes=1101004800 reduce_bytes=990904320 weight_movement_bytes=2091909120 egress_bytes=2795591840 ingress_bytes=4885207040 link_bytes=4885207040 total_nvlink_bytes=7680798880 weight_to_token_ratio=0.374298 prefetch_share_of_ingress=0.225375 dedup_ratio=0.320483 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=9 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.496178521e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5574036712 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2788164840 ingress_bytes=4657580032 link_bytes=4657580032 total_nvlink_bytes=7445744872 weight_to_token_ratio=0.335790 prefetch_share_of_ingress=0.236390 dedup_ratio=0.321951 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.100297716e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279580672 dispatch_meta_bytes=229896 combine_payload_bytes=279580672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279810568 ingress_bytes=389681152 link_bytes=389681152 token_bytes=559391240 weight_movement_bytes=110100480 total_nvlink_bytes=669491720 weight_to_token_ratio=0.196822 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777196149e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44320 payload_rows_remote=38654 dup_rows=21216 dup_rows_remote=18340 dedup_ratio=0.323730 router_tokens_per_expert=[489, 491, 493, 490, 495, 495, 533, 542, 500, 538, 516, 503, 505, 507, 529, 523, 514, 504, 486, 497, 528, 528, 501, 492, 530, 514, 480, 524, 548, 494, 535, 525, 501, 483, 559, 526, 531, 502, 535, 528, 500, 513, 507, 536, 553, 531, 545, 499, 483, 500, 480, 490, 521, 525, 529, 501, 492, 518, 495, 509, 535, 476, 494, 522] payload_rows_per_dst=[5434, 5512, 5470, 5660, 5396, 5638, 5544, 5666] dup_rows_per_dst=[2622, 2730, 2550, 2720, 2474, 2730, 2514, 2876] combine_rows_per_src=[5434, 5512, 5470, 5660, 5396, 5638, 5544, 5666] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=38996 dup_rows=21014 dup_rows_remote=18458 dedup_ratio=0.320648 router_tokens_per_expert=[550, 486, 489, 528, 506, 510, 521, 519, 536, 499, 501, 520, 502, 499, 509, 482, 499, 516, 529, 510, 495, 520, 505, 540, 504, 524, 480, 500, 518, 516, 479, 520, 476, 568, 537, 527, 545, 529, 483, 522, 501, 512, 535, 514, 522, 526, 516, 528, 480, 528, 506, 539, 534, 493, 497, 514, 510, 483, 504, 499, 497, 519, 527, 485] payload_rows_per_dst=[5536, 5568, 5602, 5526, 5606, 5624, 5536, 5524] dup_rows_per_dst=[2682, 2528, 2626, 2556, 2768, 2684, 2646, 2524] combine_rows_per_src=[5536, 5568, 5602, 5526, 5606, 5624, 5536, 5524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229120 combine_payload_bytes=278720512 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278949632 ingress_bytes=609021952 link_bytes=609021952 token_bytes=557670144 weight_movement_bytes=330301440 total_nvlink_bytes=887971584 weight_to_token_ratio=0.592288 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059189341e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44604 payload_rows_remote=39028 dup_rows=20932 dup_rows_remote=18214 dedup_ratio=0.319397 router_tokens_per_expert=[491, 494, 505, 498, 519, 502, 512, 488, 498, 458, 511, 558, 486, 522, 541, 516, 513, 505, 480, 554, 521, 516, 477, 581, 517, 517, 485, 548, 523, 516, 532, 521, 540, 479, 524, 533, 498, 502, 504, 510, 538, 494, 512, 524, 526, 491, 448, 517, 537, 530, 511, 517, 478, 526, 518, 507, 493, 515, 499, 521, 510, 497, 536, 528] payload_rows_per_dst=[5480, 5530, 5576, 5674, 5602, 5594, 5588, 5560] dup_rows_per_dst=[2538, 2650, 2718, 2644, 2578, 2506, 2660, 2638] combine_rows_per_src=[5480, 5530, 5576, 5674, 5602, 5594, 5588, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278978560 dispatch_meta_bytes=229824 combine_payload_bytes=278978560 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279208384 ingress_bytes=499179520 link_bytes=499179520 token_bytes=558186944 weight_movement_bytes=220200960 total_nvlink_bytes=778387904 weight_to_token_ratio=0.394493 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681552862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=229472 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278376544 ingress_bytes=388247552 link_bytes=388247552 token_bytes=556523616 weight_movement_bytes=110100480 total_nvlink_bytes=666624096 weight_to_token_ratio=0.197836 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752263487e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38884 dup_rows=21100 dup_rows_remote=18396 dedup_ratio=0.321960 router_tokens_per_expert=[469, 500, 514, 560, 483, 511, 497, 534, 523, 517, 491, 532, 518, 503, 508, 513, 525, 506, 518, 497, 480, 518, 508, 550, 499, 541, 523, 561, 510, 546, 475, 495, 470, 531, 500, 507, 488, 501, 480, 521, 519, 518, 459, 502, 501, 522, 510, 550, 513, 523, 528, 492, 489, 507, 495, 581, 509, 519, 515, 513, 487, 519, 566, 508] payload_rows_per_dst=[5514, 5624, 5534, 5578, 5428, 5582, 5552, 5624] dup_rows_per_dst=[2622, 2586, 2670, 2722, 2568, 2580, 2704, 2648] combine_rows_per_src=[5514, 5624, 5534, 5578, 5428, 5582, 5552, 5624] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278978560 dispatch_meta_bytes=229824 combine_payload_bytes=278978560 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279208384 ingress_bytes=499179520 link_bytes=499179520 token_bytes=558186944 weight_movement_bytes=220200960 total_nvlink_bytes=778387904 weight_to_token_ratio=0.394493 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.681552862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44614 payload_rows_remote=39016 dup_rows=20922 dup_rows_remote=18332 dedup_ratio=0.319244 router_tokens_per_expert=[511, 498, 492, 519, 495, 546, 545, 552, 498, 514, 551, 526, 511, 514, 542, 508, 480, 492, 507, 503, 547, 503, 544, 514, 477, 491, 502, 526, 470, 506, 483, 512, 498, 509, 526, 524, 515, 510, 500, 512, 510, 559, 452, 564, 509, 526, 497, 507, 498, 545, 513, 518, 490, 521, 519, 527, 522, 538, 494, 499, 505, 497, 492, 493] payload_rows_per_dst=[5714, 5662, 5538, 5472, 5598, 5492, 5592, 5546] dup_rows_per_dst=[2602, 2666, 2642, 2462, 2590, 2756, 2670, 2534] combine_rows_per_src=[5714, 5662, 5538, 5472, 5598, 5492, 5592, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44618 payload_rows_remote=38996 dup_rows=20918 dup_rows_remote=18332 dedup_ratio=0.319183 router_tokens_per_expert=[501, 517, 522, 504, 497, 522, 520, 506, 494, 518, 516, 503, 508, 507, 548, 497, 499, 527, 501, 496, 548, 490, 535, 512, 515, 488, 521, 512, 514, 532, 538, 504, 502, 518, 527, 489, 509, 531, 527, 503, 505, 538, 516, 495, 508, 480, 515, 543, 489, 523, 511, 510, 529, 538, 497, 522, 515, 486, 536, 508, 495, 517, 486, 488] payload_rows_per_dst=[5622, 5520, 5536, 5672, 5628, 5538, 5566, 5536] dup_rows_per_dst=[2586, 2634, 2684, 2576, 2702, 2550, 2660, 2526] combine_rows_per_src=[5622, 5520, 5536, 5672, 5628, 5538, 5566, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5586480192 prefetch_bytes=770703360 reduce_bytes=1101004800 weight_movement_bytes=1871708160 egress_bytes=2794386496 ingress_bytes=4663801856 link_bytes=4663801856 total_nvlink_bytes=7458188352 weight_to_token_ratio=0.335042 prefetch_share_of_ingress=0.165252 dedup_ratio=0.320523 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=10 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.111118491e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277071872 dispatch_meta_bytes=227976 combine_payload_bytes=277071872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277299848 ingress_bytes=387172352 link_bytes=387172352 token_bytes=554371720 weight_movement_bytes=110100480 total_nvlink_bytes=664472200 weight_to_token_ratio=0.198604 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.733563990e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=229816 combine_payload_bytes=279523328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279753144 ingress_bytes=389623808 link_bytes=389623808 token_bytes=559276472 weight_movement_bytes=110100480 total_nvlink_bytes=669376952 weight_to_token_ratio=0.196862 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776198842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582523712 prefetch_bytes=880803840 reduce_bytes=990904320 weight_movement_bytes=1871708160 egress_bytes=2792408384 ingress_bytes=4661823488 link_bytes=4661823488 total_nvlink_bytes=7454231872 weight_to_token_ratio=0.335280 prefetch_share_of_ingress=0.188940 dedup_ratio=0.321127 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=9 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.107677784e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=228968 combine_payload_bytes=279752704 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279981672 ingress_bytes=499953664 link_bytes=499953664 token_bytes=559734376 weight_movement_bytes=220200960 total_nvlink_bytes=779935336 weight_to_token_ratio=0.393403 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.695016499e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278720512 dispatch_meta_bytes=229120 combine_payload_bytes=278720512 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278949632 ingress_bytes=609021952 link_bytes=609021952 token_bytes=557670144 weight_movement_bytes=330301440 total_nvlink_bytes=887971584 weight_to_token_ratio=0.592288 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059189341e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582523712 prefetch_bytes=880803840 reduce_bytes=990904320 weight_movement_bytes=1871708160 egress_bytes=2792408384 ingress_bytes=4661823488 link_bytes=4661823488 total_nvlink_bytes=7454231872 weight_to_token_ratio=0.335280 prefetch_share_of_ingress=0.188940 dedup_ratio=0.321127 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=9 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.107677784e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38820 dup_rows=21094 dup_rows_remote=18436 dedup_ratio=0.321869 router_tokens_per_expert=[507, 472, 518, 530, 526, 493, 521, 505, 517, 561, 556, 509, 488, 517, 492, 500, 515, 513, 514, 521, 528, 480, 503, 535, 469, 536, 526, 487, 512, 532, 493, 495, 511, 518, 530, 538, 457, 507, 522, 512, 534, 499, 500, 497, 513, 504, 518, 526, 509, 518, 501, 499, 514, 486, 533, 504, 529, 523, 519, 539, 515, 513, 497, 512] payload_rows_per_dst=[5520, 5622, 5494, 5548, 5524, 5552, 5556, 5626] dup_rows_per_dst=[2624, 2658, 2724, 2552, 2666, 2630, 2572, 2668] combine_rows_per_src=[5520, 5622, 5494, 5548, 5524, 5552, 5556, 5626] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=229312 combine_payload_bytes=279523328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279752640 ingress_bytes=389623808 link_bytes=389623808 token_bytes=559275968 weight_movement_bytes=110100480 total_nvlink_bytes=669376448 weight_to_token_ratio=0.196863 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776198842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5570104704 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=2786196864 ingress_bytes=4325314560 link_bytes=4325314560 total_nvlink_bytes=7111511424 weight_to_token_ratio=0.276729 prefetch_share_of_ingress=0.229094 dedup_ratio=0.321811 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.522433412e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=229392 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279896080 ingress_bytes=499867648 link_bytes=499867648 token_bytes=559562768 weight_movement_bytes=220200960 total_nvlink_bytes=779763728 weight_to_token_ratio=0.393523 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.693520540e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44604 payload_rows_remote=39028 dup_rows=20932 dup_rows_remote=18214 dedup_ratio=0.319397 router_tokens_per_expert=[491, 494, 505, 498, 519, 502, 512, 488, 498, 458, 511, 558, 486, 522, 541, 516, 513, 505, 480, 554, 521, 516, 477, 581, 517, 517, 485, 548, 523, 516, 532, 521, 540, 479, 524, 533, 498, 502, 504, 510, 538, 494, 512, 524, 526, 491, 448, 517, 537, 530, 511, 517, 478, 526, 518, 507, 493, 515, 499, 521, 510, 497, 536, 528] payload_rows_per_dst=[5480, 5530, 5576, 5674, 5602, 5594, 5588, 5560] dup_rows_per_dst=[2538, 2650, 2718, 2644, 2578, 2506, 2660, 2638] combine_rows_per_src=[5480, 5530, 5576, 5674, 5602, 5594, 5588, 5560] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5594023512 prefetch_bytes=880803840 reduce_bytes=1101004800 weight_movement_bytes=1981808640 egress_bytes=2798159448 ingress_bytes=4777672704 link_bytes=4777672704 total_nvlink_bytes=7575832152 weight_to_token_ratio=0.354272 prefetch_share_of_ingress=0.184358 dedup_ratio=0.320065 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=10 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.309158624e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5586738768 prefetch_bytes=880803840 reduce_bytes=1541406720 weight_movement_bytes=2422210560 egress_bytes=2794516048 ingress_bytes=5214433280 link_bytes=5214433280 total_nvlink_bytes=8008949328 weight_to_token_ratio=0.433564 prefetch_share_of_ingress=0.168917 dedup_ratio=0.320554 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=14 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.068757101e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5586480192 prefetch_bytes=770703360 reduce_bytes=1101004800 weight_movement_bytes=1871708160 egress_bytes=2794386496 ingress_bytes=4663801856 link_bytes=4663801856 total_nvlink_bytes=7458188352 weight_to_token_ratio=0.335042 prefetch_share_of_ingress=0.165252 dedup_ratio=0.320523 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=10 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.111118491e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5578734600 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=2790511624 ingress_bytes=4549830656 link_bytes=4549830656 total_nvlink_bytes=7340342280 weight_to_token_ratio=0.315772 prefetch_share_of_ingress=0.241988 dedup_ratio=0.320773 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.912903829e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none - [2026-08-26 16:58:54] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 357.0 | throughput per GPU (TFLOP/s/GPU): 345.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.535656E+00 | loss scale: 1.0 | grad norm: 4.066 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44438 payload_rows_remote=38886 dup_rows=21098 dup_rows_remote=18418 dedup_ratio=0.321930 router_tokens_per_expert=[487, 510, 501, 516, 517, 476, 497, 526, 488, 491, 521, 537, 513, 511, 527, 528, 585, 516, 515, 513, 543, 523, 525, 538, 507, 501, 518, 505, 524, 504, 507, 493, 540, 537, 522, 512, 520, 532, 507, 480, 514, 483, 492, 521, 503, 511, 473, 486, 528, 513, 520, 497, 470, 528, 509, 493, 505, 532, 531, 516, 501, 486, 548, 495] payload_rows_per_dst=[5504, 5552, 5694, 5506, 5622, 5498, 5498, 5564] dup_rows_per_dst=[2556, 2680, 2822, 2612, 2678, 2468, 2618, 2664] combine_rows_per_src=[5504, 5552, 5694, 5506, 5622, 5498, 5498, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278261760 dispatch_meta_bytes=229024 combine_payload_bytes=278261760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278490784 ingress_bytes=498462720 link_bytes=498462720 token_bytes=556752544 weight_movement_bytes=220200960 total_nvlink_bytes=776953504 weight_to_token_ratio=0.395510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.669086531e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5574036712 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2788164840 ingress_bytes=4657580032 link_bytes=4657580032 total_nvlink_bytes=7445744872 weight_to_token_ratio=0.335790 prefetch_share_of_ingress=0.236390 dedup_ratio=0.321951 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.100297716e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279752704 dispatch_meta_bytes=228968 combine_payload_bytes=279752704 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279981672 ingress_bytes=499953664 link_bytes=499953664 token_bytes=559734376 weight_movement_bytes=220200960 total_nvlink_bytes=779935336 weight_to_token_ratio=0.393403 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.695016499e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5588889760 prefetch_bytes=1101004800 reduce_bytes=990904320 weight_movement_bytes=2091909120 egress_bytes=2795591840 ingress_bytes=4885207040 link_bytes=4885207040 total_nvlink_bytes=7680798880 weight_to_token_ratio=0.374298 prefetch_share_of_ingress=0.225375 dedup_ratio=0.320483 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=9 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.496178521e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5586738768 prefetch_bytes=880803840 reduce_bytes=1541406720 weight_movement_bytes=2422210560 egress_bytes=2794516048 ingress_bytes=5214433280 link_bytes=5214433280 total_nvlink_bytes=8008949328 weight_to_token_ratio=0.433564 prefetch_share_of_ingress=0.168917 dedup_ratio=0.320554 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=14 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.068757101e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none + [2026-08-26 18:10:59] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 361.2 | throughput per GPU (TFLOP/s/GPU): 341.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.534000E+00 | loss scale: 1.0 | grad norm: 3.941 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38924 dup_rows=21150 dup_rows_remote=18528 dedup_ratio=0.322723 router_tokens_per_expert=[499, 512, 539, 525, 515, 514, 536, 511, 518, 517, 522, 515, 523, 517, 525, 533, 544, 500, 515, 500, 501, 515, 524, 521, 462, 533, 512, 507, 485, 504, 521, 518, 525, 515, 541, 476, 491, 521, 510, 537, 495, 567, 516, 452, 523, 507, 539, 521, 481, 504, 544, 486, 507, 507, 501, 510, 499, 506, 476, 511, 504, 521, 509, 483] payload_rows_per_dst=[5562, 5562, 5594, 5462, 5574, 5570, 5520, 5542] dup_rows_per_dst=[2740, 2778, 2646, 2622, 2658, 2670, 2560, 2476] combine_rows_per_src=[5562, 5562, 5594, 5462, 5574, 5570, 5520, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38822 dup_rows=21064 dup_rows_remote=18418 dedup_ratio=0.321411 router_tokens_per_expert=[493, 493, 501, 531, 559, 509, 507, 528, 538, 513, 514, 498, 477, 528, 485, 542, 534, 517, 535, 495, 528, 519, 532, 541, 487, 523, 499, 501, 524, 482, 512, 496, 514, 541, 505, 502, 526, 490, 473, 502, 486, 533, 534, 519, 504, 508, 486, 501, 547, 517, 521, 528, 537, 539, 487, 472, 504, 507, 501, 494, 528, 509, 506, 506] payload_rows_per_dst=[5610, 5496, 5638, 5534, 5568, 5512, 5650, 5464] dup_rows_per_dst=[2632, 2694, 2764, 2514, 2538, 2630, 2646, 2646] combine_rows_per_src=[5610, 5496, 5638, 5534, 5568, 5512, 5650, 5464] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44580 payload_rows_remote=39022 dup_rows=20956 dup_rows_remote=18272 dedup_ratio=0.319763 router_tokens_per_expert=[502, 554, 506, 531, 476, 478, 520, 546, 515, 517, 516, 522, 513, 498, 522, 507, 498, 459, 500, 507, 529, 525, 486, 494, 508, 499, 534, 503, 520, 532, 491, 494, 511, 487, 516, 490, 538, 543, 530, 529, 505, 510, 533, 504, 550, 490, 484, 545, 497, 510, 516, 491, 554, 489, 505, 511, 506, 549, 517, 523, 535, 518, 484, 496] payload_rows_per_dst=[5584, 5624, 5468, 5638, 5574, 5558, 5568, 5566] dup_rows_per_dst=[2642, 2596, 2528, 2524, 2714, 2684, 2578, 2690] combine_rows_per_src=[5584, 5624, 5468, 5638, 5574, 5558, 5568, 5566] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44416 payload_rows_remote=38950 dup_rows=21120 dup_rows_remote=18524 dedup_ratio=0.322266 router_tokens_per_expert=[536, 521, 515, 517, 530, 501, 507, 539, 523, 474, 476, 520, 479, 504, 516, 526, 476, 476, 509, 530, 517, 506, 514, 506, 483, 534, 499, 553, 509, 499, 519, 536, 507, 509, 528, 525, 479, 508, 490, 485, 548, 523, 485, 531, 532, 518, 560, 521, 484, 463, 567, 520, 492, 512, 503, 533, 519, 479, 468, 506, 521, 530, 533, 539] payload_rows_per_dst=[5598, 5488, 5484, 5628, 5466, 5662, 5530, 5560] dup_rows_per_dst=[2734, 2548, 2584, 2636, 2596, 2774, 2618, 2630] combine_rows_per_src=[5598, 5488, 5484, 5628, 5466, 5662, 5530, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38924 dup_rows=21150 dup_rows_remote=18528 dedup_ratio=0.322723 router_tokens_per_expert=[499, 512, 539, 525, 515, 514, 536, 511, 518, 517, 522, 515, 523, 517, 525, 533, 544, 500, 515, 500, 501, 515, 524, 521, 462, 533, 512, 507, 485, 504, 521, 518, 525, 515, 541, 476, 491, 521, 510, 537, 495, 567, 516, 452, 523, 507, 539, 521, 481, 504, 544, 486, 507, 507, 501, 510, 499, 506, 476, 511, 504, 521, 509, 483] payload_rows_per_dst=[5562, 5562, 5594, 5462, 5574, 5570, 5520, 5542] dup_rows_per_dst=[2740, 2778, 2646, 2622, 2658, 2670, 2560, 2476] combine_rows_per_src=[5562, 5562, 5594, 5462, 5574, 5570, 5520, 5542] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44624 payload_rows_remote=38970 dup_rows=20912 dup_rows_remote=18276 dedup_ratio=0.319092 router_tokens_per_expert=[523, 510, 535, 511, 482, 503, 493, 518, 515, 486, 530, 528, 504, 508, 475, 526, 518, 515, 520, 510, 512, 537, 495, 503, 556, 518, 522, 523, 498, 482, 545, 506, 471, 520, 497, 550, 497, 522, 486, 502, 479, 530, 517, 478, 534, 500, 512, 500, 509, 494, 554, 516, 547, 518, 514, 472, 499, 514, 543, 546, 497, 495, 534, 514] payload_rows_per_dst=[5498, 5498, 5606, 5668, 5508, 5498, 5694, 5654] dup_rows_per_dst=[2376, 2578, 2682, 2908, 2364, 2596, 2772, 2636] combine_rows_per_src=[5498, 5498, 5606, 5668, 5508, 5498, 5694, 5654] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44472 payload_rows_remote=38822 dup_rows=21064 dup_rows_remote=18418 dedup_ratio=0.321411 router_tokens_per_expert=[493, 493, 501, 531, 559, 509, 507, 528, 538, 513, 514, 498, 477, 528, 485, 542, 534, 517, 535, 495, 528, 519, 532, 541, 487, 523, 499, 501, 524, 482, 512, 496, 514, 541, 505, 502, 526, 490, 473, 502, 486, 533, 534, 519, 504, 508, 486, 501, 547, 517, 521, 528, 537, 539, 487, 472, 504, 507, 501, 494, 528, 509, 506, 506] payload_rows_per_dst=[5610, 5496, 5638, 5534, 5568, 5512, 5650, 5464] dup_rows_per_dst=[2632, 2694, 2764, 2514, 2538, 2630, 2646, 2646] combine_rows_per_src=[5610, 5496, 5638, 5534, 5568, 5512, 5650, 5464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44640 payload_rows_remote=39084 dup_rows=20896 dup_rows_remote=18382 dedup_ratio=0.318848 router_tokens_per_expert=[451, 484, 557, 506, 518, 513, 522, 531, 508, 521, 510, 546, 532, 473, 521, 508, 489, 517, 497, 500, 514, 505, 494, 519, 518, 534, 494, 528, 516, 507, 523, 473, 496, 525, 535, 500, 513, 534, 514, 501, 516, 481, 524, 515, 573, 528, 468, 557, 520, 492, 500, 478, 542, 501, 508, 494, 522, 493, 498, 514, 523, 528, 532, 514] payload_rows_per_dst=[5554, 5622, 5556, 5582, 5618, 5654, 5490, 5564] dup_rows_per_dst=[2610, 2616, 2514, 2604, 2618, 2670, 2580, 2684] combine_rows_per_src=[5554, 5622, 5556, 5582, 5618, 5654, 5490, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44438 payload_rows_remote=38886 dup_rows=21098 dup_rows_remote=18418 dedup_ratio=0.321930 router_tokens_per_expert=[487, 510, 501, 516, 517, 476, 497, 526, 488, 491, 521, 537, 513, 511, 527, 528, 585, 516, 515, 513, 543, 523, 525, 538, 507, 501, 518, 505, 524, 504, 507, 493, 540, 537, 522, 512, 520, 532, 507, 480, 514, 483, 492, 521, 503, 511, 473, 486, 528, 513, 520, 497, 470, 528, 509, 493, 505, 532, 531, 516, 501, 486, 548, 495] payload_rows_per_dst=[5504, 5552, 5694, 5506, 5622, 5498, 5498, 5564] dup_rows_per_dst=[2556, 2680, 2822, 2612, 2678, 2468, 2618, 2664] combine_rows_per_src=[5504, 5552, 5694, 5506, 5622, 5498, 5498, 5564] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44382 payload_rows_remote=38786 dup_rows=21154 dup_rows_remote=18414 dedup_ratio=0.322784 router_tokens_per_expert=[491, 513, 529, 501, 513, 512, 525, 541, 516, 499, 501, 503, 539, 530, 500, 513, 500, 491, 537, 460, 442, 506, 512, 530, 527, 488, 509, 481, 537, 490, 509, 508, 510, 517, 483, 523, 536, 501, 554, 545, 537, 518, 542, 479, 502, 478, 507, 535, 528, 566, 477, 488, 542, 516, 483, 507, 511, 528, 487, 537, 533, 516, 499, 530] payload_rows_per_dst=[5596, 5594, 5470, 5506, 5582, 5490, 5602, 5542] dup_rows_per_dst=[2740, 2610, 2486, 2592, 2858, 2602, 2612, 2654] combine_rows_per_src=[5596, 5594, 5470, 5506, 5582, 5490, 5602, 5542] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=229216 combine_payload_bytes=278734848 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278964064 ingress_bytes=498935808 link_bytes=498935808 token_bytes=557698912 weight_movement_bytes=220200960 total_nvlink_bytes=777899872 weight_to_token_ratio=0.394838 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677314310e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229808 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279237040 ingress_bytes=389107712 link_bytes=389107712 token_bytes=558244272 weight_movement_bytes=110100480 total_nvlink_bytes=668344752 weight_to_token_ratio=0.197226 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767223084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279193600 dispatch_meta_bytes=229896 combine_payload_bytes=279193600 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279423496 ingress_bytes=499394560 link_bytes=499394560 token_bytes=558617096 weight_movement_bytes=220200960 total_nvlink_bytes=778818056 weight_to_token_ratio=0.394189 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685292761e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278276096 dispatch_meta_bytes=228960 combine_payload_bytes=278276096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278505056 ingress_bytes=388376576 link_bytes=388376576 token_bytes=556781152 weight_movement_bytes=110100480 total_nvlink_bytes=666881632 weight_to_token_ratio=0.197745 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754507426e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279709696 dispatch_meta_bytes=229176 combine_payload_bytes=279709696 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279938872 ingress_bytes=610011136 link_bytes=610011136 token_bytes=559648568 weight_movement_bytes=330301440 total_nvlink_bytes=889950008 weight_to_token_ratio=0.590194 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060909695e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279193600 dispatch_meta_bytes=229896 combine_payload_bytes=279193600 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279423496 ingress_bytes=499394560 link_bytes=499394560 token_bytes=558617096 weight_movement_bytes=220200960 total_nvlink_bytes=778818056 weight_to_token_ratio=0.394189 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685292761e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279336960 dispatch_meta_bytes=228984 combine_payload_bytes=279336960 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279565944 ingress_bytes=499537920 link_bytes=499537920 token_bytes=558902904 weight_movement_bytes=220200960 total_nvlink_bytes=779103864 weight_to_token_ratio=0.393988 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.687786027e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=229216 combine_payload_bytes=278734848 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278964064 ingress_bytes=498935808 link_bytes=498935808 token_bytes=557698912 weight_movement_bytes=220200960 total_nvlink_bytes=777899872 weight_to_token_ratio=0.394838 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.677314310e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278276096 dispatch_meta_bytes=228960 combine_payload_bytes=278276096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278505056 ingress_bytes=388376576 link_bytes=388376576 token_bytes=556781152 weight_movement_bytes=110100480 total_nvlink_bytes=666881632 weight_to_token_ratio=0.197745 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.754507426e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280154112 dispatch_meta_bytes=229864 combine_payload_bytes=280154112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280383976 ingress_bytes=390254592 link_bytes=390254592 token_bytes=560538088 weight_movement_bytes=110100480 total_nvlink_bytes=670638568 weight_to_token_ratio=0.196419 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787169213e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44640 payload_rows_remote=39084 dup_rows=20896 dup_rows_remote=18382 dedup_ratio=0.318848 router_tokens_per_expert=[451, 484, 557, 506, 518, 513, 522, 531, 508, 521, 510, 546, 532, 473, 521, 508, 489, 517, 497, 500, 514, 505, 494, 519, 518, 534, 494, 528, 516, 507, 523, 473, 496, 525, 535, 500, 513, 534, 514, 501, 516, 481, 524, 515, 573, 528, 468, 557, 520, 492, 500, 478, 542, 501, 508, 494, 522, 493, 498, 514, 523, 528, 532, 514] payload_rows_per_dst=[5554, 5622, 5556, 5582, 5618, 5654, 5490, 5564] dup_rows_per_dst=[2610, 2616, 2514, 2604, 2618, 2670, 2580, 2684] combine_rows_per_src=[5554, 5622, 5556, 5582, 5618, 5654, 5490, 5564] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278018048 dispatch_meta_bytes=228800 combine_payload_bytes=278018048 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278246848 ingress_bytes=498219008 link_bytes=498219008 token_bytes=556264896 weight_movement_bytes=220200960 total_nvlink_bytes=776465856 weight_to_token_ratio=0.395856 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.664847979e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44380 payload_rows_remote=38770 dup_rows=21156 dup_rows_remote=18540 dedup_ratio=0.322815 router_tokens_per_expert=[484, 511, 489, 515, 492, 545, 494, 539, 539, 494, 501, 515, 539, 500, 535, 488, 519, 522, 499, 561, 485, 524, 464, 491, 530, 514, 490, 481, 520, 537, 484, 506, 505, 514, 524, 519, 513, 497, 520, 521, 499, 533, 521, 552, 524, 505, 550, 483, 484, 522, 528, 550, 518, 484, 479, 511, 542, 519, 496, 460, 519, 540, 547, 482] payload_rows_per_dst=[5538, 5608, 5524, 5480, 5610, 5580, 5510, 5530] dup_rows_per_dst=[2600, 2614, 2606, 2644, 2616, 2754, 2642, 2680] combine_rows_per_src=[5538, 5608, 5524, 5480, 5610, 5580, 5510, 5530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38958 dup_rows=20984 dup_rows_remote=18256 dedup_ratio=0.320190 router_tokens_per_expert=[484, 505, 480, 529, 473, 518, 519, 530, 519, 529, 507, 540, 531, 494, 503, 538, 496, 497, 534, 496, 511, 518, 580, 548, 486, 500, 511, 513, 504, 495, 490, 506, 498, 497, 522, 519, 534, 517, 537, 515, 543, 489, 533, 489, 485, 494, 520, 518, 502, 526, 495, 506, 504, 504, 512, 511, 495, 519, 529, 529, 522, 491, 532, 497] payload_rows_per_dst=[5540, 5594, 5682, 5522, 5650, 5478, 5470, 5616] dup_rows_per_dst=[2536, 2728, 2678, 2488, 2628, 2664, 2650, 2612] combine_rows_per_src=[5540, 5594, 5682, 5522, 5650, 5478, 5470, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38928 dup_rows=21010 dup_rows_remote=18292 dedup_ratio=0.320587 router_tokens_per_expert=[490, 536, 478, 544, 522, 525, 537, 511, 515, 520, 469, 513, 497, 486, 489, 512, 491, 522, 512, 510, 513, 558, 546, 519, 517, 520, 534, 521, 501, 511, 531, 523, 535, 526, 496, 473, 543, 543, 521, 502, 479, 496, 474, 510, 503, 496, 519, 516, 488, 514, 529, 545, 527, 461, 531, 527, 496, 449, 545, 499, 541, 532, 499, 480] payload_rows_per_dst=[5576, 5572, 5628, 5598, 5574, 5450, 5628, 5500] dup_rows_per_dst=[2710, 2430, 2714, 2718, 2704, 2536, 2616, 2582] combine_rows_per_src=[5576, 5572, 5628, 5598, 5574, 5450, 5628, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44396 payload_rows_remote=38888 dup_rows=21140 dup_rows_remote=18564 dedup_ratio=0.322571 router_tokens_per_expert=[511, 503, 491, 538, 530, 514, 491, 524, 497, 505, 548, 508, 520, 537, 532, 511, 507, 502, 505, 507, 505, 499, 497, 484, 508, 497, 535, 533, 518, 526, 539, 512, 514, 518, 539, 520, 494, 518, 550, 489, 504, 520, 496, 512, 510, 513, 465, 526, 486, 499, 528, 511, 523, 486, 511, 498, 511, 518, 509, 500, 499, 497, 551, 519] payload_rows_per_dst=[5494, 5516, 5502, 5578, 5634, 5602, 5508, 5562] dup_rows_per_dst=[2710, 2800, 2510, 2758, 2650, 2490, 2576, 2646] combine_rows_per_src=[5494, 5516, 5502, 5578, 5634, 5602, 5508, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44380 payload_rows_remote=38770 dup_rows=21156 dup_rows_remote=18540 dedup_ratio=0.322815 router_tokens_per_expert=[484, 511, 489, 515, 492, 545, 494, 539, 539, 494, 501, 515, 539, 500, 535, 488, 519, 522, 499, 561, 485, 524, 464, 491, 530, 514, 490, 481, 520, 537, 484, 506, 505, 514, 524, 519, 513, 497, 520, 521, 499, 533, 521, 552, 524, 505, 550, 483, 484, 522, 528, 550, 518, 484, 479, 511, 542, 519, 496, 460, 519, 540, 547, 482] payload_rows_per_dst=[5538, 5608, 5524, 5480, 5610, 5580, 5510, 5530] dup_rows_per_dst=[2600, 2614, 2606, 2644, 2616, 2754, 2642, 2680] combine_rows_per_src=[5538, 5608, 5524, 5480, 5610, 5580, 5510, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280154112 dispatch_meta_bytes=229864 combine_payload_bytes=280154112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280383976 ingress_bytes=390254592 link_bytes=390254592 token_bytes=560538088 weight_movement_bytes=110100480 total_nvlink_bytes=670638568 weight_to_token_ratio=0.196419 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787169213e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44440 payload_rows_remote=39026 dup_rows=21096 dup_rows_remote=18628 dedup_ratio=0.321899 router_tokens_per_expert=[550, 492, 491, 536, 497, 508, 549, 517, 522, 504, 498, 501, 490, 492, 498, 516, 522, 514, 493, 546, 518, 500, 560, 517, 512, 543, 517, 498, 511, 486, 483, 474, 527, 540, 533, 514, 537, 523, 519, 520, 479, 512, 499, 494, 529, 492, 450, 486, 495, 487, 564, 496, 515, 520, 504, 507, 542, 492, 544, 509, 545, 529, 486, 524] payload_rows_per_dst=[5600, 5472, 5546, 5528, 5646, 5414, 5518, 5716] dup_rows_per_dst=[2680, 2570, 2794, 2520, 2780, 2468, 2658, 2626] combine_rows_per_src=[5600, 5472, 5546, 5528, 5646, 5414, 5518, 5716] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44346 payload_rows_remote=38686 dup_rows=21190 dup_rows_remote=18496 dedup_ratio=0.323334 router_tokens_per_expert=[510, 504, 503, 530, 514, 522, 513, 489, 561, 484, 482, 520, 484, 527, 532, 487, 492, 478, 477, 519, 525, 492, 512, 498, 508, 526, 480, 524, 502, 525, 505, 531, 505, 496, 503, 538, 492, 544, 517, 519, 520, 508, 530, 487, 502, 526, 549, 531, 511, 533, 475, 512, 499, 539, 529, 508, 493, 551, 502, 499, 585, 516, 468, 525] payload_rows_per_dst=[5596, 5506, 5458, 5534, 5544, 5516, 5532, 5660] dup_rows_per_dst=[2574, 2648, 2514, 2668, 2688, 2790, 2614, 2694] combine_rows_per_src=[5596, 5506, 5458, 5534, 5544, 5516, 5532, 5660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44552 payload_rows_remote=38958 dup_rows=20984 dup_rows_remote=18256 dedup_ratio=0.320190 router_tokens_per_expert=[484, 505, 480, 529, 473, 518, 519, 530, 519, 529, 507, 540, 531, 494, 503, 538, 496, 497, 534, 496, 511, 518, 580, 548, 486, 500, 511, 513, 504, 495, 490, 506, 498, 497, 522, 519, 534, 517, 537, 515, 543, 489, 533, 489, 485, 494, 520, 518, 502, 526, 495, 506, 504, 504, 512, 511, 495, 519, 529, 529, 522, 491, 532, 497] payload_rows_per_dst=[5540, 5594, 5682, 5522, 5650, 5478, 5470, 5616] dup_rows_per_dst=[2536, 2728, 2678, 2488, 2628, 2664, 2650, 2612] combine_rows_per_src=[5540, 5594, 5682, 5522, 5650, 5478, 5470, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44396 payload_rows_remote=38888 dup_rows=21140 dup_rows_remote=18564 dedup_ratio=0.322571 router_tokens_per_expert=[511, 503, 491, 538, 530, 514, 491, 524, 497, 505, 548, 508, 520, 537, 532, 511, 507, 502, 505, 507, 505, 499, 497, 484, 508, 497, 535, 533, 518, 526, 539, 512, 514, 518, 539, 520, 494, 518, 550, 489, 504, 520, 496, 512, 510, 513, 465, 526, 486, 499, 528, 511, 523, 486, 511, 498, 511, 518, 509, 500, 499, 497, 551, 519] payload_rows_per_dst=[5494, 5516, 5502, 5578, 5634, 5602, 5508, 5562] dup_rows_per_dst=[2710, 2800, 2510, 2758, 2650, 2490, 2576, 2646] combine_rows_per_src=[5494, 5516, 5502, 5578, 5634, 5602, 5508, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277903360 dispatch_meta_bytes=229240 combine_payload_bytes=277903360 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278132600 ingress_bytes=498104320 link_bytes=498104320 token_bytes=556035960 weight_movement_bytes=220200960 total_nvlink_bytes=776236920 weight_to_token_ratio=0.396019 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662853366e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44318 payload_rows_remote=38700 dup_rows=21218 dup_rows_remote=18510 dedup_ratio=0.323761 router_tokens_per_expert=[480, 497, 479, 545, 526, 514, 508, 466, 557, 526, 503, 573, 548, 543, 514, 513, 539, 537, 518, 473, 500, 553, 529, 514, 535, 509, 488, 545, 526, 477, 474, 480, 504, 496, 492, 487, 453, 490, 505, 483, 528, 497, 500, 555, 543, 511, 487, 539, 537, 562, 493, 505, 485, 540, 513, 510, 508, 498, 523, 498, 517, 506, 523, 491] payload_rows_per_dst=[5522, 5590, 5618, 5492, 5372, 5620, 5550, 5554] dup_rows_per_dst=[2508, 2964, 2708, 2576, 2448, 2700, 2740, 2574] combine_rows_per_src=[5522, 5590, 5618, 5492, 5372, 5620, 5550, 5554] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=228856 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279479800 ingress_bytes=499451904 link_bytes=499451904 token_bytes=558730744 weight_movement_bytes=220200960 total_nvlink_bytes=778931704 weight_to_token_ratio=0.394109 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686290068e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279035904 dispatch_meta_bytes=228880 combine_payload_bytes=279035904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279264784 ingress_bytes=389136384 link_bytes=389136384 token_bytes=558300688 weight_movement_bytes=110100480 total_nvlink_bytes=668401168 weight_to_token_ratio=0.197206 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767721737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44382 payload_rows_remote=38792 dup_rows=21154 dup_rows_remote=18392 dedup_ratio=0.322784 router_tokens_per_expert=[502, 489, 491, 503, 501, 515, 494, 488, 524, 525, 535, 540, 509, 457, 507, 514, 476, 470, 505, 527, 527, 514, 503, 505, 526, 530, 502, 516, 549, 486, 497, 509, 511, 453, 526, 533, 547, 512, 514, 501, 528, 520, 502, 536, 488, 508, 504, 499, 527, 505, 505, 526, 558, 501, 538, 546, 517, 496, 518, 502, 522, 527, 528, 534] payload_rows_per_dst=[5590, 5416, 5486, 5632, 5498, 5674, 5600, 5486] dup_rows_per_dst=[2762, 2508, 2568, 2800, 2494, 2800, 2724, 2498] combine_rows_per_src=[5590, 5416, 5486, 5632, 5498, 5674, 5600, 5486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277903360 dispatch_meta_bytes=229240 combine_payload_bytes=277903360 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278132600 ingress_bytes=498104320 link_bytes=498104320 token_bytes=556035960 weight_movement_bytes=220200960 total_nvlink_bytes=776236920 weight_to_token_ratio=0.396019 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.662853366e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229808 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278978992 ingress_bytes=609050624 link_bytes=609050624 token_bytes=557728176 weight_movement_bytes=330301440 total_nvlink_bytes=888029616 weight_to_token_ratio=0.592227 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059239207e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279035904 dispatch_meta_bytes=228880 combine_payload_bytes=279035904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279264784 ingress_bytes=389136384 link_bytes=389136384 token_bytes=558300688 weight_movement_bytes=110100480 total_nvlink_bytes=668401168 weight_to_token_ratio=0.197206 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767721737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279738368 dispatch_meta_bytes=230616 combine_payload_bytes=279738368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279968984 ingress_bytes=389838848 link_bytes=389838848 token_bytes=559707352 weight_movement_bytes=110100480 total_nvlink_bytes=669807832 weight_to_token_ratio=0.196711 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779938741e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277301248 dispatch_meta_bytes=228728 combine_payload_bytes=277301248 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277529976 ingress_bytes=497502208 link_bytes=497502208 token_bytes=554831224 weight_movement_bytes=220200960 total_nvlink_bytes=775032184 weight_to_token_ratio=0.396879 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652381648e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=228856 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279479800 ingress_bytes=499451904 link_bytes=499451904 token_bytes=558730744 weight_movement_bytes=220200960 total_nvlink_bytes=778931704 weight_to_token_ratio=0.394109 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686290068e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229808 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278978992 ingress_bytes=609050624 link_bytes=609050624 token_bytes=557728176 weight_movement_bytes=330301440 total_nvlink_bytes=888029616 weight_to_token_ratio=0.592227 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059239207e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277401600 dispatch_meta_bytes=228840 combine_payload_bytes=277401600 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277630440 ingress_bytes=497602560 link_bytes=497602560 token_bytes=555032040 weight_movement_bytes=220200960 total_nvlink_bytes=775233000 weight_to_token_ratio=0.396736 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.654126934e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278061056 dispatch_meta_bytes=228736 combine_payload_bytes=278061056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278289792 ingress_bytes=388161536 link_bytes=388161536 token_bytes=556350848 weight_movement_bytes=110100480 total_nvlink_bytes=666451328 weight_to_token_ratio=0.197898 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750767527e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44248 payload_rows_remote=38688 dup_rows=21288 dup_rows_remote=18704 dedup_ratio=0.324829 router_tokens_per_expert=[533, 491, 498, 469, 496, 505, 508, 515, 562, 500, 526, 512, 548, 532, 520, 478, 495, 534, 500, 522, 498, 513, 518, 532, 493, 500, 533, 512, 518, 550, 494, 531, 510, 501, 524, 510, 495, 530, 495, 507, 503, 523, 548, 508, 532, 504, 505, 537, 528, 501, 507, 486, 526, 503, 501, 480, 511, 500, 461, 513, 519, 540, 514, 510] payload_rows_per_dst=[5438, 5634, 5570, 5558, 5560, 5494, 5442, 5552] dup_rows_per_dst=[2592, 2722, 2654, 2704, 2584, 2826, 2622, 2584] combine_rows_per_src=[5438, 5634, 5570, 5558, 5560, 5494, 5442, 5552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44732 payload_rows_remote=39104 dup_rows=20804 dup_rows_remote=18224 dedup_ratio=0.317444 router_tokens_per_expert=[507, 515, 492, 515, 520, 485, 482, 509, 525, 514, 546, 528, 486, 483, 519, 503, 517, 493, 501, 484, 522, 535, 525, 520, 517, 510, 484, 519, 505, 538, 536, 499, 534, 529, 505, 466, 499, 541, 547, 544, 513, 493, 531, 521, 494, 505, 530, 544, 533, 480, 488, 513, 550, 475, 493, 506, 510, 524, 484, 513, 499, 510, 520, 540] payload_rows_per_dst=[5576, 5628, 5538, 5546, 5642, 5672, 5544, 5586] dup_rows_per_dst=[2474, 2580, 2656, 2670, 2688, 2590, 2532, 2614] combine_rows_per_src=[5576, 5628, 5538, 5546, 5642, 5672, 5544, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44318 payload_rows_remote=38700 dup_rows=21218 dup_rows_remote=18510 dedup_ratio=0.323761 router_tokens_per_expert=[480, 497, 479, 545, 526, 514, 508, 466, 557, 526, 503, 573, 548, 543, 514, 513, 539, 537, 518, 473, 500, 553, 529, 514, 535, 509, 488, 545, 526, 477, 474, 480, 504, 496, 492, 487, 453, 490, 505, 483, 528, 497, 500, 555, 543, 511, 487, 539, 537, 562, 493, 505, 485, 540, 513, 510, 508, 498, 523, 498, 517, 506, 523, 491] payload_rows_per_dst=[5522, 5590, 5618, 5492, 5372, 5620, 5550, 5554] dup_rows_per_dst=[2508, 2964, 2708, 2576, 2448, 2700, 2740, 2574] combine_rows_per_src=[5522, 5590, 5618, 5492, 5372, 5620, 5550, 5554] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38908 dup_rows=20966 dup_rows_remote=18242 dedup_ratio=0.319916 router_tokens_per_expert=[507, 531, 488, 498, 492, 474, 497, 517, 528, 534, 488, 488, 502, 532, 522, 517, 492, 489, 505, 518, 478, 497, 514, 488, 524, 546, 481, 529, 560, 528, 514, 511, 529, 503, 514, 525, 515, 517, 543, 507, 485, 502, 500, 519, 512, 494, 493, 522, 532, 499, 546, 524, 494, 510, 512, 529, 533, 493, 511, 510, 532, 517, 520, 537] payload_rows_per_dst=[5524, 5574, 5492, 5662, 5600, 5508, 5628, 5582] dup_rows_per_dst=[2484, 2648, 2470, 2724, 2706, 2546, 2664, 2724] combine_rows_per_src=[5524, 5574, 5492, 5662, 5600, 5508, 5628, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=39040 dup_rows=21000 dup_rows_remote=18262 dedup_ratio=0.320435 router_tokens_per_expert=[492, 554, 532, 518, 531, 508, 457, 501, 500, 510, 517, 518, 489, 536, 538, 513, 506, 491, 537, 513, 534, 487, 503, 517, 526, 499, 499, 493, 482, 503, 514, 490, 492, 497, 527, 529, 498, 488, 558, 497, 554, 521, 512, 462, 515, 491, 517, 545, 528, 540, 555, 546, 517, 525, 490, 487, 486, 497, 517, 524, 534, 511, 513, 487] payload_rows_per_dst=[5586, 5684, 5530, 5444, 5554, 5496, 5654, 5588] dup_rows_per_dst=[2600, 2558, 2646, 2568, 2618, 2738, 2722, 2550] combine_rows_per_src=[5586, 5684, 5530, 5444, 5554, 5496, 5654, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44248 payload_rows_remote=38688 dup_rows=21288 dup_rows_remote=18704 dedup_ratio=0.324829 router_tokens_per_expert=[533, 491, 498, 469, 496, 505, 508, 515, 562, 500, 526, 512, 548, 532, 520, 478, 495, 534, 500, 522, 498, 513, 518, 532, 493, 500, 533, 512, 518, 550, 494, 531, 510, 501, 524, 510, 495, 530, 495, 507, 503, 523, 548, 508, 532, 504, 505, 537, 528, 501, 507, 486, 526, 503, 501, 480, 511, 500, 461, 513, 519, 540, 514, 510] payload_rows_per_dst=[5438, 5634, 5570, 5558, 5560, 5494, 5442, 5552] dup_rows_per_dst=[2592, 2722, 2654, 2704, 2584, 2826, 2622, 2584] combine_rows_per_src=[5438, 5634, 5570, 5558, 5560, 5494, 5442, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38910 dup_rows=21138 dup_rows_remote=18490 dedup_ratio=0.322540 router_tokens_per_expert=[477, 505, 547, 479, 526, 502, 470, 530, 557, 503, 520, 480, 525, 474, 564, 513, 527, 529, 501, 530, 509, 504, 512, 499, 490, 516, 508, 503, 510, 529, 498, 522, 523, 526, 490, 537, 546, 510, 533, 474, 528, 490, 516, 498, 510, 513, 505, 517, 501, 526, 517, 499, 489, 479, 525, 532, 533, 523, 488, 492, 549, 536, 508, 496] payload_rows_per_dst=[5424, 5652, 5570, 5590, 5582, 5500, 5488, 5592] dup_rows_per_dst=[2648, 2620, 2652, 2562, 2696, 2654, 2648, 2658] combine_rows_per_src=[5424, 5652, 5570, 5590, 5582, 5500, 5488, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44506 payload_rows_remote=38804 dup_rows=21030 dup_rows_remote=18026 dedup_ratio=0.320892 router_tokens_per_expert=[474, 528, 488, 480, 517, 534, 527, 496, 481, 546, 509, 500, 547, 513, 486, 467, 514, 536, 515, 509, 507, 499, 538, 510, 476, 509, 519, 529, 500, 502, 493, 505, 519, 509, 505, 529, 517, 503, 486, 486, 476, 495, 475, 518, 540, 547, 526, 545, 549, 532, 497, 527, 575, 496, 501, 498, 505, 510, 520, 542, 516, 500, 569, 501] payload_rows_per_dst=[5532, 5502, 5604, 5536, 5514, 5506, 5610, 5702] dup_rows_per_dst=[2556, 2596, 2652, 2402, 2594, 2486, 2740, 3004] combine_rows_per_src=[5532, 5502, 5604, 5536, 5514, 5506, 5610, 5702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=39040 dup_rows=21000 dup_rows_remote=18262 dedup_ratio=0.320435 router_tokens_per_expert=[492, 554, 532, 518, 531, 508, 457, 501, 500, 510, 517, 518, 489, 536, 538, 513, 506, 491, 537, 513, 534, 487, 503, 517, 526, 499, 499, 493, 482, 503, 514, 490, 492, 497, 527, 529, 498, 488, 558, 497, 554, 521, 512, 462, 515, 491, 517, 545, 528, 540, 555, 546, 517, 525, 490, 487, 486, 497, 517, 524, 534, 511, 513, 487] payload_rows_per_dst=[5586, 5684, 5530, 5444, 5554, 5496, 5654, 5588] dup_rows_per_dst=[2600, 2558, 2646, 2568, 2618, 2738, 2722, 2550] combine_rows_per_src=[5586, 5684, 5530, 5444, 5554, 5496, 5654, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44732 payload_rows_remote=39104 dup_rows=20804 dup_rows_remote=18224 dedup_ratio=0.317444 router_tokens_per_expert=[507, 515, 492, 515, 520, 485, 482, 509, 525, 514, 546, 528, 486, 483, 519, 503, 517, 493, 501, 484, 522, 535, 525, 520, 517, 510, 484, 519, 505, 538, 536, 499, 534, 529, 505, 466, 499, 541, 547, 544, 513, 493, 531, 521, 494, 505, 530, 544, 533, 480, 488, 513, 550, 475, 493, 506, 510, 524, 484, 513, 499, 510, 520, 540] payload_rows_per_dst=[5576, 5628, 5538, 5546, 5642, 5672, 5544, 5586] dup_rows_per_dst=[2474, 2580, 2656, 2670, 2688, 2590, 2532, 2614] combine_rows_per_src=[5576, 5628, 5538, 5546, 5642, 5672, 5544, 5586] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38910 dup_rows=21138 dup_rows_remote=18490 dedup_ratio=0.322540 router_tokens_per_expert=[477, 505, 547, 479, 526, 502, 470, 530, 557, 503, 520, 480, 525, 474, 564, 513, 527, 529, 501, 530, 509, 504, 512, 499, 490, 516, 508, 503, 510, 529, 498, 522, 523, 526, 490, 537, 546, 510, 533, 474, 528, 490, 516, 498, 510, 513, 505, 517, 501, 526, 517, 499, 489, 479, 525, 532, 533, 523, 488, 492, 549, 536, 508, 496] payload_rows_per_dst=[5424, 5652, 5570, 5590, 5582, 5500, 5488, 5592] dup_rows_per_dst=[2648, 2620, 2652, 2562, 2696, 2654, 2648, 2658] combine_rows_per_src=[5424, 5652, 5570, 5590, 5582, 5500, 5488, 5592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277315584 dispatch_meta_bytes=229568 combine_payload_bytes=277315584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277545152 ingress_bytes=497516544 link_bytes=497516544 token_bytes=554860736 weight_movement_bytes=220200960 total_nvlink_bytes=775061696 weight_to_token_ratio=0.396858 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652630974e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280297472 dispatch_meta_bytes=229312 combine_payload_bytes=280297472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280526784 ingress_bytes=390397952 link_bytes=390397952 token_bytes=560824256 weight_movement_bytes=110100480 total_nvlink_bytes=670924736 weight_to_token_ratio=0.196319 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789662480e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=39048 dup_rows=21000 dup_rows_remote=18390 dedup_ratio=0.320435 router_tokens_per_expert=[494, 501, 488, 493, 531, 506, 521, 539, 521, 475, 533, 487, 479, 516, 476, 496, 530, 527, 494, 502, 503, 488, 501, 504, 542, 504, 531, 496, 533, 480, 537, 516, 517, 549, 508, 491, 526, 496, 515, 501, 546, 541, 507, 546, 506, 509, 534, 494, 498, 483, 510, 499, 505, 511, 551, 522, 513, 494, 530, 528, 515, 545, 528, 506] payload_rows_per_dst=[5612, 5442, 5488, 5574, 5568, 5656, 5592, 5604] dup_rows_per_dst=[2534, 2524, 2610, 2704, 2638, 2710, 2566, 2714] combine_rows_per_src=[5612, 5442, 5488, 5574, 5568, 5656, 5592, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278892544 dispatch_meta_bytes=228600 combine_payload_bytes=278892544 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279121144 ingress_bytes=499093504 link_bytes=499093504 token_bytes=558013688 weight_movement_bytes=220200960 total_nvlink_bytes=778214648 weight_to_token_ratio=0.394616 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680056902e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277401600 dispatch_meta_bytes=228840 combine_payload_bytes=277401600 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=277630440 ingress_bytes=497602560 link_bytes=497602560 token_bytes=555032040 weight_movement_bytes=220200960 total_nvlink_bytes=775233000 weight_to_token_ratio=0.396736 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.654126934e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44452 payload_rows_remote=38562 dup_rows=21084 dup_rows_remote=18018 dedup_ratio=0.321716 router_tokens_per_expert=[511, 519, 542, 507, 514, 494, 495, 520, 505, 500, 514, 495, 501, 519, 518, 518, 487, 506, 519, 468, 500, 517, 466, 511, 531, 536, 485, 512, 516, 513, 498, 541, 546, 515, 508, 522, 499, 544, 514, 491, 480, 539, 518, 488, 495, 523, 499, 535, 505, 499, 526, 500, 512, 528, 555, 532, 518, 530, 534, 504, 508, 520, 490, 513] payload_rows_per_dst=[5890, 5480, 5674, 5606, 5434, 5544, 5540, 5284] dup_rows_per_dst=[3066, 2692, 2730, 2686, 2558, 2610, 2544, 2198] combine_rows_per_src=[5890, 5480, 5674, 5606, 5434, 5544, 5540, 5284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229208 combine_payload_bytes=279838720 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280067928 ingress_bytes=500039680 link_bytes=500039680 token_bytes=559906648 weight_movement_bytes=220200960 total_nvlink_bytes=780107608 weight_to_token_ratio=0.393282 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.696512459e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=227320 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278374392 ingress_bytes=498348032 link_bytes=498348032 token_bytes=556521464 weight_movement_bytes=220200960 total_nvlink_bytes=776722424 weight_to_token_ratio=0.395674 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.667091918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278892544 dispatch_meta_bytes=228600 combine_payload_bytes=278892544 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279121144 ingress_bytes=499093504 link_bytes=499093504 token_bytes=558013688 weight_movement_bytes=220200960 total_nvlink_bytes=778214648 weight_to_token_ratio=0.394616 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680056902e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277315584 dispatch_meta_bytes=229568 combine_payload_bytes=277315584 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277545152 ingress_bytes=497516544 link_bytes=497516544 token_bytes=554860736 weight_movement_bytes=220200960 total_nvlink_bytes=775061696 weight_to_token_ratio=0.396858 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652630974e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278906880 dispatch_meta_bytes=229600 combine_payload_bytes=278906880 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279136480 ingress_bytes=609208320 link_bytes=609208320 token_bytes=558043360 weight_movement_bytes=330301440 total_nvlink_bytes=888344800 weight_to_token_ratio=0.591892 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059513466e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=227320 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278374392 ingress_bytes=498348032 link_bytes=498348032 token_bytes=556521464 weight_movement_bytes=220200960 total_nvlink_bytes=776722424 weight_to_token_ratio=0.395674 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.667091918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279896064 dispatch_meta_bytes=229752 combine_payload_bytes=279896064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280125816 ingress_bytes=389996544 link_bytes=389996544 token_bytes=560021880 weight_movement_bytes=110100480 total_nvlink_bytes=670122360 weight_to_token_ratio=0.196600 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782681334e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279838720 dispatch_meta_bytes=229208 combine_payload_bytes=279838720 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280067928 ingress_bytes=500039680 link_bytes=500039680 token_bytes=559906648 weight_movement_bytes=220200960 total_nvlink_bytes=780107608 weight_to_token_ratio=0.393282 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.696512459e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280297472 dispatch_meta_bytes=229312 combine_payload_bytes=280297472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280526784 ingress_bytes=390397952 link_bytes=390397952 token_bytes=560824256 weight_movement_bytes=110100480 total_nvlink_bytes=670924736 weight_to_token_ratio=0.196319 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789662480e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276412416 dispatch_meta_bytes=226320 combine_payload_bytes=276412416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276638736 ingress_bytes=386512896 link_bytes=386512896 token_bytes=553051152 weight_movement_bytes=110100480 total_nvlink_bytes=663151632 weight_to_token_ratio=0.199078 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.722094966e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44480 payload_rows_remote=38868 dup_rows=21056 dup_rows_remote=18450 dedup_ratio=0.321289 router_tokens_per_expert=[498, 531, 483, 507, 502, 519, 525, 486, 507, 523, 508, 490, 499, 533, 523, 493, 528, 534, 536, 483, 544, 476, 546, 536, 502, 500, 540, 517, 519, 511, 508, 495, 510, 538, 505, 494, 513, 506, 512, 531, 534, 546, 485, 478, 528, 524, 469, 499, 538, 477, 542, 546, 550, 510, 486, 513, 504, 520, 531, 556, 476, 481, 467, 497] payload_rows_per_dst=[5560, 5552, 5620, 5558, 5612, 5512, 5626, 5440] dup_rows_per_dst=[2542, 2600, 2746, 2626, 2606, 2614, 2698, 2624] combine_rows_per_src=[5560, 5552, 5620, 5558, 5612, 5512, 5626, 5440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=39102 dup_rows=20934 dup_rows_remote=18296 dedup_ratio=0.319427 router_tokens_per_expert=[510, 534, 529, 513, 552, 531, 497, 497, 501, 537, 481, 540, 479, 514, 521, 496, 531, 540, 499, 537, 481, 532, 500, 514, 501, 515, 502, 510, 461, 549, 526, 491, 542, 532, 521, 493, 536, 507, 481, 470, 486, 512, 505, 496, 481, 560, 534, 501, 473, 511, 519, 497, 491, 530, 523, 502, 518, 510, 515, 532, 549, 511, 503, 506] payload_rows_per_dst=[5664, 5500, 5654, 5528, 5536, 5586, 5526, 5608] dup_rows_per_dst=[2662, 2638, 2614, 2582, 2628, 2564, 2566, 2680] combine_rows_per_src=[5664, 5500, 5654, 5528, 5536, 5586, 5526, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=39048 dup_rows=21000 dup_rows_remote=18390 dedup_ratio=0.320435 router_tokens_per_expert=[494, 501, 488, 493, 531, 506, 521, 539, 521, 475, 533, 487, 479, 516, 476, 496, 530, 527, 494, 502, 503, 488, 501, 504, 542, 504, 531, 496, 533, 480, 537, 516, 517, 549, 508, 491, 526, 496, 515, 501, 546, 541, 507, 546, 506, 509, 534, 494, 498, 483, 510, 499, 505, 511, 551, 522, 513, 494, 530, 528, 515, 545, 528, 506] payload_rows_per_dst=[5612, 5442, 5488, 5574, 5568, 5656, 5592, 5604] dup_rows_per_dst=[2534, 2524, 2610, 2704, 2638, 2710, 2566, 2714] combine_rows_per_src=[5612, 5442, 5488, 5574, 5568, 5656, 5592, 5604] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44374 payload_rows_remote=38762 dup_rows=21162 dup_rows_remote=18548 dedup_ratio=0.322906 router_tokens_per_expert=[520, 516, 525, 507, 501, 550, 547, 504, 514, 507, 509, 529, 516, 513, 515, 532, 483, 498, 521, 487, 535, 499, 545, 508, 525, 553, 527, 515, 531, 494, 453, 515, 500, 498, 496, 527, 530, 498, 544, 509, 542, 530, 486, 446, 521, 491, 512, 508, 522, 470, 526, 484, 501, 534, 536, 542, 482, 501, 551, 493, 487, 516, 494, 497] payload_rows_per_dst=[5608, 5528, 5552, 5612, 5524, 5492, 5542, 5516] dup_rows_per_dst=[2732, 2742, 2600, 2614, 2680, 2580, 2688, 2526] combine_rows_per_src=[5608, 5528, 5552, 5612, 5524, 5492, 5542, 5516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=229272 combine_payload_bytes=278605824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278835096 ingress_bytes=388706304 link_bytes=388706304 token_bytes=557440920 weight_movement_bytes=110100480 total_nvlink_bytes=667541400 weight_to_token_ratio=0.197511 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760241939e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38960 dup_rows=20978 dup_rows_remote=18278 dedup_ratio=0.320099 router_tokens_per_expert=[516, 535, 495, 497, 541, 491, 503, 545, 500, 529, 532, 519, 508, 524, 506, 513, 454, 523, 516, 539, 527, 509, 484, 539, 539, 513, 485, 487, 523, 508, 505, 514, 502, 513, 505, 521, 540, 528, 540, 496, 485, 544, 507, 518, 497, 550, 493, 555, 508, 477, 513, 496, 511, 534, 487, 509, 508, 495, 522, 500, 484, 490, 538, 483] payload_rows_per_dst=[5654, 5626, 5542, 5548, 5576, 5598, 5546, 5468] dup_rows_per_dst=[2592, 2636, 2640, 2600, 2714, 2700, 2524, 2572] combine_rows_per_src=[5654, 5626, 5542, 5548, 5576, 5598, 5546, 5468] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38782 dup_rows=21116 dup_rows_remote=18176 dedup_ratio=0.322205 router_tokens_per_expert=[503, 543, 510, 505, 521, 469, 515, 502, 498, 529, 533, 502, 524, 520, 546, 507, 531, 546, 525, 484, 530, 525, 522, 513, 494, 483, 542, 518, 538, 488, 487, 522, 522, 496, 502, 475, 509, 493, 535, 459, 494, 535, 493, 503, 515, 519, 516, 525, 496, 523, 520, 541, 511, 470, 525, 498, 520, 511, 508, 544, 492, 555, 472, 516] payload_rows_per_dst=[5414, 5636, 5454, 5602, 5576, 5572, 5528, 5638] dup_rows_per_dst=[2490, 2666, 2372, 2660, 2638, 2694, 2656, 2940] combine_rows_per_src=[5414, 5636, 5454, 5602, 5576, 5572, 5528, 5638] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44480 payload_rows_remote=38868 dup_rows=21056 dup_rows_remote=18450 dedup_ratio=0.321289 router_tokens_per_expert=[498, 531, 483, 507, 502, 519, 525, 486, 507, 523, 508, 490, 499, 533, 523, 493, 528, 534, 536, 483, 544, 476, 546, 536, 502, 500, 540, 517, 519, 511, 508, 495, 510, 538, 505, 494, 513, 506, 512, 531, 534, 546, 485, 478, 528, 524, 469, 499, 538, 477, 542, 546, 550, 510, 486, 513, 504, 520, 531, 556, 476, 481, 467, 497] payload_rows_per_dst=[5560, 5552, 5620, 5558, 5612, 5512, 5626, 5440] dup_rows_per_dst=[2542, 2600, 2746, 2626, 2606, 2614, 2698, 2624] combine_rows_per_src=[5560, 5552, 5620, 5558, 5612, 5512, 5626, 5440] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44506 payload_rows_remote=38946 dup_rows=21030 dup_rows_remote=18416 dedup_ratio=0.320892 router_tokens_per_expert=[509, 502, 526, 494, 542, 503, 485, 492, 521, 522, 505, 483, 487, 493, 523, 499, 516, 519, 524, 510, 518, 498, 535, 508, 535, 510, 500, 523, 527, 498, 548, 516, 508, 486, 502, 494, 517, 492, 534, 506, 510, 552, 501, 530, 517, 510, 525, 488, 539, 501, 508, 488, 557, 496, 503, 495, 561, 479, 516, 521, 497, 526, 517, 521] payload_rows_per_dst=[5580, 5502, 5550, 5618, 5506, 5616, 5560, 5574] dup_rows_per_dst=[2526, 2564, 2706, 2696, 2572, 2650, 2614, 2702] combine_rows_per_src=[5580, 5502, 5550, 5618, 5506, 5616, 5560, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44420 payload_rows_remote=38782 dup_rows=21116 dup_rows_remote=18176 dedup_ratio=0.322205 router_tokens_per_expert=[503, 543, 510, 505, 521, 469, 515, 502, 498, 529, 533, 502, 524, 520, 546, 507, 531, 546, 525, 484, 530, 525, 522, 513, 494, 483, 542, 518, 538, 488, 487, 522, 522, 496, 502, 475, 509, 493, 535, 459, 494, 535, 493, 503, 515, 519, 516, 525, 496, 523, 520, 541, 511, 470, 525, 498, 520, 511, 508, 544, 492, 555, 472, 516] payload_rows_per_dst=[5414, 5636, 5454, 5602, 5576, 5572, 5528, 5638] dup_rows_per_dst=[2490, 2666, 2372, 2660, 2638, 2694, 2656, 2940] combine_rows_per_src=[5414, 5636, 5454, 5602, 5576, 5572, 5528, 5638] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38776 dup_rows=21126 dup_rows_remote=18404 dedup_ratio=0.322357 router_tokens_per_expert=[505, 515, 535, 534, 517, 532, 523, 473, 505, 494, 505, 536, 520, 513, 510, 518, 526, 543, 491, 520, 536, 510, 539, 513, 471, 527, 492, 491, 499, 540, 510, 509, 536, 525, 521, 460, 502, 519, 492, 514, 514, 496, 509, 514, 510, 478, 496, 519, 531, 530, 530, 487, 514, 511, 512, 506, 504, 510, 493, 491, 529, 503, 516, 544] payload_rows_per_dst=[5534, 5588, 5634, 5500, 5548, 5460, 5606, 5540] dup_rows_per_dst=[2734, 2614, 2722, 2578, 2590, 2612, 2636, 2640] combine_rows_per_src=[5534, 5588, 5634, 5500, 5548, 5460, 5606, 5540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=229592 combine_payload_bytes=280283136 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=280512728 ingress_bytes=390383616 link_bytes=390383616 token_bytes=560795864 weight_movement_bytes=110100480 total_nvlink_bytes=670896344 weight_to_token_ratio=0.196329 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789413153e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44602 payload_rows_remote=39102 dup_rows=20934 dup_rows_remote=18296 dedup_ratio=0.319427 router_tokens_per_expert=[510, 534, 529, 513, 552, 531, 497, 497, 501, 537, 481, 540, 479, 514, 521, 496, 531, 540, 499, 537, 481, 532, 500, 514, 501, 515, 502, 510, 461, 549, 526, 491, 542, 532, 521, 493, 536, 507, 481, 470, 486, 512, 505, 496, 481, 560, 534, 501, 473, 511, 519, 497, 491, 530, 523, 502, 518, 510, 515, 532, 549, 511, 503, 506] payload_rows_per_dst=[5664, 5500, 5654, 5528, 5536, 5586, 5526, 5608] dup_rows_per_dst=[2662, 2638, 2614, 2582, 2628, 2564, 2566, 2680] combine_rows_per_src=[5664, 5500, 5654, 5528, 5536, 5586, 5526, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=38960 dup_rows=20978 dup_rows_remote=18278 dedup_ratio=0.320099 router_tokens_per_expert=[516, 535, 495, 497, 541, 491, 503, 545, 500, 529, 532, 519, 508, 524, 506, 513, 454, 523, 516, 539, 527, 509, 484, 539, 539, 513, 485, 487, 523, 508, 505, 514, 502, 513, 505, 521, 540, 528, 540, 496, 485, 544, 507, 518, 497, 550, 493, 555, 508, 477, 513, 496, 511, 534, 487, 509, 508, 495, 522, 500, 484, 490, 538, 483] payload_rows_per_dst=[5654, 5626, 5542, 5548, 5576, 5598, 5546, 5468] dup_rows_per_dst=[2592, 2636, 2640, 2600, 2714, 2700, 2524, 2572] combine_rows_per_src=[5654, 5626, 5542, 5548, 5576, 5598, 5546, 5468] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277846016 dispatch_meta_bytes=229240 combine_payload_bytes=277846016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278075256 ingress_bytes=387946496 link_bytes=387946496 token_bytes=555921272 weight_movement_bytes=110100480 total_nvlink_bytes=666021752 weight_to_token_ratio=0.198050 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747027628e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=228952 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279494232 ingress_bytes=389365760 link_bytes=389365760 token_bytes=558759512 weight_movement_bytes=110100480 total_nvlink_bytes=668859992 weight_to_token_ratio=0.197044 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771710963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277989376 dispatch_meta_bytes=227832 combine_payload_bytes=277989376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278217208 ingress_bytes=388089856 link_bytes=388089856 token_bytes=556206584 weight_movement_bytes=110100480 total_nvlink_bytes=666307064 weight_to_token_ratio=0.197949 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749520894e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38868 dup_rows=21076 dup_rows_remote=18424 dedup_ratio=0.321594 router_tokens_per_expert=[576, 536, 481, 460, 511, 506, 489, 526, 513, 552, 499, 474, 485, 514, 535, 500, 533, 500, 479, 537, 486, 480, 517, 514, 528, 526, 510, 513, 490, 542, 507, 491, 532, 493, 527, 508, 558, 485, 507, 505, 508, 531, 506, 509, 501, 537, 521, 530, 532, 519, 515, 537, 507, 526, 511, 519, 498, 478, 517, 488, 516, 516, 529, 492] payload_rows_per_dst=[5592, 5582, 5556, 5488, 5588, 5646, 5548, 5460] dup_rows_per_dst=[2652, 2562, 2574, 2726, 2642, 2640, 2672, 2608] combine_rows_per_src=[5592, 5582, 5556, 5488, 5588, 5646, 5548, 5460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279896064 dispatch_meta_bytes=229752 combine_payload_bytes=279896064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280125816 ingress_bytes=389996544 link_bytes=389996544 token_bytes=560021880 weight_movement_bytes=110100480 total_nvlink_bytes=670122360 weight_to_token_ratio=0.196600 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.782681334e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279164928 dispatch_meta_bytes=229448 combine_payload_bytes=279164928 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279394376 ingress_bytes=609466368 link_bytes=609466368 token_bytes=558559304 weight_movement_bytes=330301440 total_nvlink_bytes=888860744 weight_to_token_ratio=0.591345 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059962254e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277989376 dispatch_meta_bytes=227832 combine_payload_bytes=277989376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278217208 ingress_bytes=388089856 link_bytes=388089856 token_bytes=556206584 weight_movement_bytes=110100480 total_nvlink_bytes=666307064 weight_to_token_ratio=0.197949 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749520894e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38868 dup_rows=21076 dup_rows_remote=18424 dedup_ratio=0.321594 router_tokens_per_expert=[576, 536, 481, 460, 511, 506, 489, 526, 513, 552, 499, 474, 485, 514, 535, 500, 533, 500, 479, 537, 486, 480, 517, 514, 528, 526, 510, 513, 490, 542, 507, 491, 532, 493, 527, 508, 558, 485, 507, 505, 508, 531, 506, 509, 501, 537, 521, 530, 532, 519, 515, 537, 507, 526, 511, 519, 498, 478, 517, 488, 516, 516, 529, 492] payload_rows_per_dst=[5592, 5582, 5556, 5488, 5588, 5646, 5548, 5460] dup_rows_per_dst=[2652, 2562, 2574, 2726, 2642, 2640, 2672, 2608] combine_rows_per_src=[5592, 5582, 5556, 5488, 5588, 5646, 5548, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=229272 combine_payload_bytes=278605824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278835096 ingress_bytes=388706304 link_bytes=388706304 token_bytes=557440920 weight_movement_bytes=110100480 total_nvlink_bytes=667541400 weight_to_token_ratio=0.197511 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760241939e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228720 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278175088 ingress_bytes=718348288 link_bytes=718348288 token_bytes=556121456 weight_movement_bytes=440401920 total_nvlink_bytes=996523376 weight_to_token_ratio=0.791917 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249325821e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38806 dup_rows=21168 dup_rows_remote=18474 dedup_ratio=0.322998 router_tokens_per_expert=[540, 546, 496, 522, 463, 542, 549, 481, 512, 500, 495, 560, 532, 513, 548, 491, 515, 531, 499, 474, 525, 527, 496, 511, 546, 477, 513, 507, 524, 482, 542, 500, 528, 517, 504, 488, 573, 505, 504, 509, 489, 503, 487, 527, 509, 516, 511, 501, 523, 553, 466, 480, 523, 501, 555, 494, 505, 501, 529, 491, 506, 509, 513, 489] payload_rows_per_dst=[5588, 5640, 5504, 5550, 5562, 5504, 5542, 5478] dup_rows_per_dst=[2690, 2662, 2652, 2632, 2694, 2582, 2648, 2608] combine_rows_per_src=[5588, 5640, 5504, 5550, 5562, 5504, 5542, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=229592 combine_payload_bytes=280283136 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=280512728 ingress_bytes=390383616 link_bytes=390383616 token_bytes=560795864 weight_movement_bytes=110100480 total_nvlink_bytes=670896344 weight_to_token_ratio=0.196329 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789413153e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279265280 dispatch_meta_bytes=228952 combine_payload_bytes=279265280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279494232 ingress_bytes=389365760 link_bytes=389365760 token_bytes=558759512 weight_movement_bytes=110100480 total_nvlink_bytes=668859992 weight_to_token_ratio=0.197044 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771710963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=229168 combine_payload_bytes=278605824 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278834992 ingress_bytes=498806784 link_bytes=498806784 token_bytes=557440816 weight_movement_bytes=220200960 total_nvlink_bytes=777641776 weight_to_token_ratio=0.395021 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.675070370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=39016 dup_rows=20978 dup_rows_remote=18338 dedup_ratio=0.320099 router_tokens_per_expert=[489, 528, 507, 505, 522, 506, 554, 495, 529, 551, 509, 493, 524, 515, 508, 462, 542, 486, 528, 518, 495, 491, 481, 499, 560, 521, 511, 498, 511, 514, 509, 507, 506, 519, 522, 546, 473, 480, 522, 495, 508, 509, 481, 521, 533, 503, 556, 527, 513, 526, 544, 473, 524, 530, 524, 497, 512, 490, 516, 535, 500, 520, 518, 477] payload_rows_per_dst=[5636, 5542, 5542, 5550, 5544, 5574, 5642, 5528] dup_rows_per_dst=[2576, 2640, 2538, 2712, 2582, 2702, 2620, 2608] combine_rows_per_src=[5636, 5542, 5542, 5550, 5544, 5574, 5642, 5528] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44748 payload_rows_remote=39078 dup_rows=20788 dup_rows_remote=18150 dedup_ratio=0.317200 router_tokens_per_expert=[496, 514, 512, 534, 490, 518, 516, 499, 478, 503, 475, 524, 510, 536, 503, 522, 510, 491, 481, 500, 479, 536, 527, 503, 511, 532, 489, 511, 525, 520, 536, 530, 522, 513, 497, 513, 492, 525, 467, 516, 524, 552, 481, 525, 523, 508, 491, 507, 504, 512, 490, 492, 575, 536, 555, 499, 522, 556, 516, 552, 484, 507, 496, 505] payload_rows_per_dst=[5548, 5636, 5524, 5670, 5556, 5618, 5568, 5628] dup_rows_per_dst=[2610, 2466, 2530, 2638, 2534, 2604, 2758, 2648] combine_rows_per_src=[5548, 5636, 5524, 5670, 5556, 5618, 5568, 5628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278161408 dispatch_meta_bytes=229120 combine_payload_bytes=278161408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278390528 ingress_bytes=388261888 link_bytes=388261888 token_bytes=556551936 weight_movement_bytes=110100480 total_nvlink_bytes=666652416 weight_to_token_ratio=0.197826 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752512813e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38974 dup_rows=21052 dup_rows_remote=18494 dedup_ratio=0.321228 router_tokens_per_expert=[513, 527, 487, 506, 482, 503, 513, 501, 531, 509, 488, 508, 543, 510, 512, 511, 500, 533, 497, 499, 527, 501, 541, 511, 529, 516, 522, 532, 531, 503, 518, 521, 489, 495, 515, 517, 474, 505, 553, 500, 499, 487, 480, 504, 512, 493, 526, 533, 531, 503, 526, 533, 513, 516, 493, 529, 515, 517, 495, 501, 544, 510, 507, 528] payload_rows_per_dst=[5518, 5568, 5530, 5586, 5588, 5510, 5652, 5532] dup_rows_per_dst=[2546, 2656, 2688, 2758, 2508, 2558, 2636, 2702] combine_rows_per_src=[5518, 5568, 5530, 5586, 5588, 5510, 5652, 5532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44372 payload_rows_remote=38728 dup_rows=21164 dup_rows_remote=18464 dedup_ratio=0.322937 router_tokens_per_expert=[489, 490, 507, 501, 494, 519, 479, 514, 565, 482, 527, 498, 508, 466, 551, 565, 471, 535, 507, 506, 538, 516, 494, 530, 524, 484, 525, 503, 493, 514, 532, 529, 506, 514, 525, 499, 496, 531, 516, 532, 505, 500, 538, 549, 501, 523, 495, 515, 494, 503, 499, 479, 477, 496, 508, 539, 505, 515, 530, 510, 507, 549, 513, 543] payload_rows_per_dst=[5496, 5578, 5472, 5556, 5602, 5592, 5432, 5644] dup_rows_per_dst=[2490, 2746, 2722, 2652, 2636, 2660, 2558, 2700] combine_rows_per_src=[5496, 5578, 5472, 5556, 5602, 5592, 5432, 5644] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44524 payload_rows_remote=39022 dup_rows=21012 dup_rows_remote=18308 dedup_ratio=0.320618 router_tokens_per_expert=[527, 501, 526, 489, 483, 518, 509, 571, 515, 513, 520, 480, 533, 522, 506, 487, 506, 530, 527, 529, 523, 494, 514, 510, 513, 485, 507, 517, 510, 509, 520, 467, 536, 526, 514, 502, 517, 499, 515, 490, 530, 500, 497, 521, 522, 530, 516, 500, 508, 502, 489, 499, 564, 508, 506, 527, 521, 509, 535, 502, 471, 492, 559, 500] payload_rows_per_dst=[5590, 5602, 5604, 5508, 5556, 5590, 5502, 5572] dup_rows_per_dst=[2658, 2550, 2662, 2548, 2642, 2642, 2704, 2606] combine_rows_per_src=[5590, 5602, 5604, 5508, 5556, 5590, 5502, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44410 payload_rows_remote=38776 dup_rows=21126 dup_rows_remote=18404 dedup_ratio=0.322357 router_tokens_per_expert=[505, 515, 535, 534, 517, 532, 523, 473, 505, 494, 505, 536, 520, 513, 510, 518, 526, 543, 491, 520, 536, 510, 539, 513, 471, 527, 492, 491, 499, 540, 510, 509, 536, 525, 521, 460, 502, 519, 492, 514, 514, 496, 509, 514, 510, 478, 496, 519, 531, 530, 530, 487, 514, 511, 512, 506, 504, 510, 493, 491, 529, 503, 516, 544] payload_rows_per_dst=[5534, 5588, 5634, 5500, 5548, 5460, 5606, 5540] dup_rows_per_dst=[2734, 2614, 2722, 2578, 2590, 2612, 2636, 2640] combine_rows_per_src=[5534, 5588, 5634, 5500, 5548, 5460, 5606, 5540] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=229416 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279896104 ingress_bytes=389767168 link_bytes=389767168 token_bytes=559562792 weight_movement_bytes=110100480 total_nvlink_bytes=669663272 weight_to_token_ratio=0.196762 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778692108e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44372 payload_rows_remote=38728 dup_rows=21164 dup_rows_remote=18464 dedup_ratio=0.322937 router_tokens_per_expert=[489, 490, 507, 501, 494, 519, 479, 514, 565, 482, 527, 498, 508, 466, 551, 565, 471, 535, 507, 506, 538, 516, 494, 530, 524, 484, 525, 503, 493, 514, 532, 529, 506, 514, 525, 499, 496, 531, 516, 532, 505, 500, 538, 549, 501, 523, 495, 515, 494, 503, 499, 479, 477, 496, 508, 539, 505, 515, 530, 510, 507, 549, 513, 543] payload_rows_per_dst=[5496, 5578, 5472, 5556, 5602, 5592, 5432, 5644] dup_rows_per_dst=[2490, 2746, 2722, 2652, 2636, 2660, 2558, 2700] combine_rows_per_src=[5496, 5578, 5472, 5556, 5602, 5592, 5432, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38806 dup_rows=21168 dup_rows_remote=18474 dedup_ratio=0.322998 router_tokens_per_expert=[540, 546, 496, 522, 463, 542, 549, 481, 512, 500, 495, 560, 532, 513, 548, 491, 515, 531, 499, 474, 525, 527, 496, 511, 546, 477, 513, 507, 524, 482, 542, 500, 528, 517, 504, 488, 573, 505, 504, 509, 489, 503, 487, 527, 509, 516, 511, 501, 523, 553, 466, 480, 523, 501, 555, 494, 505, 501, 529, 491, 506, 509, 513, 489] payload_rows_per_dst=[5588, 5640, 5504, 5550, 5562, 5504, 5542, 5478] dup_rows_per_dst=[2690, 2662, 2652, 2632, 2694, 2582, 2648, 2608] combine_rows_per_src=[5588, 5640, 5504, 5550, 5562, 5504, 5542, 5478] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44772 payload_rows_remote=39206 dup_rows=20764 dup_rows_remote=18334 dedup_ratio=0.316833 router_tokens_per_expert=[488, 492, 529, 527, 550, 495, 470, 495, 502, 529, 490, 522, 488, 527, 505, 488, 475, 497, 490, 504, 501, 518, 520, 493, 531, 514, 543, 511, 510, 497, 492, 512, 528, 497, 506, 584, 505, 542, 530, 517, 540, 500, 492, 494, 537, 507, 538, 518, 511, 534, 545, 515, 532, 529, 498, 485, 521, 516, 509, 482, 474, 510, 515, 552] payload_rows_per_dst=[5544, 5554, 5566, 5592, 5670, 5584, 5648, 5614] dup_rows_per_dst=[2548, 2548, 2430, 2628, 2748, 2668, 2650, 2544] combine_rows_per_src=[5544, 5554, 5566, 5592, 5670, 5584, 5648, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280111104 dispatch_meta_bytes=228912 combine_payload_bytes=280111104 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280340016 ingress_bytes=500312064 link_bytes=500312064 token_bytes=560451120 weight_movement_bytes=220200960 total_nvlink_bytes=780652080 weight_to_token_ratio=0.392899 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701249665e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279365632 dispatch_meta_bytes=229872 combine_payload_bytes=279365632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279595504 ingress_bytes=389466112 link_bytes=389466112 token_bytes=558961136 weight_movement_bytes=110100480 total_nvlink_bytes=669061616 weight_to_token_ratio=0.196973 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773456249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38724 dup_rows=21166 dup_rows_remote=18314 dedup_ratio=0.322968 router_tokens_per_expert=[520, 526, 533, 487, 495, 491, 514, 524, 483, 519, 536, 496, 511, 517, 481, 529, 533, 488, 506, 497, 497, 544, 505, 506, 505, 516, 529, 487, 514, 518, 529, 536, 524, 477, 546, 525, 530, 468, 484, 492, 473, 504, 493, 499, 511, 502, 506, 516, 530, 515, 517, 538, 495, 538, 542, 474, 514, 542, 517, 528, 514, 518, 543, 521] payload_rows_per_dst=[5646, 5584, 5624, 5446, 5484, 5520, 5494, 5572] dup_rows_per_dst=[2852, 2564, 2948, 2510, 2630, 2628, 2482, 2552] combine_rows_per_src=[5646, 5584, 5624, 5446, 5484, 5520, 5494, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38974 dup_rows=21052 dup_rows_remote=18494 dedup_ratio=0.321228 router_tokens_per_expert=[513, 527, 487, 506, 482, 503, 513, 501, 531, 509, 488, 508, 543, 510, 512, 511, 500, 533, 497, 499, 527, 501, 541, 511, 529, 516, 522, 532, 531, 503, 518, 521, 489, 495, 515, 517, 474, 505, 553, 500, 499, 487, 480, 504, 512, 493, 526, 533, 531, 503, 526, 533, 513, 516, 493, 529, 515, 517, 495, 501, 544, 510, 507, 528] payload_rows_per_dst=[5518, 5568, 5530, 5586, 5588, 5510, 5652, 5532] dup_rows_per_dst=[2546, 2656, 2688, 2758, 2508, 2558, 2636, 2702] combine_rows_per_src=[5518, 5568, 5530, 5586, 5588, 5510, 5652, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44558 payload_rows_remote=39016 dup_rows=20978 dup_rows_remote=18338 dedup_ratio=0.320099 router_tokens_per_expert=[489, 528, 507, 505, 522, 506, 554, 495, 529, 551, 509, 493, 524, 515, 508, 462, 542, 486, 528, 518, 495, 491, 481, 499, 560, 521, 511, 498, 511, 514, 509, 507, 506, 519, 522, 546, 473, 480, 522, 495, 508, 509, 481, 521, 533, 503, 556, 527, 513, 526, 544, 473, 524, 530, 524, 497, 512, 490, 516, 535, 500, 520, 518, 477] payload_rows_per_dst=[5636, 5542, 5542, 5550, 5544, 5574, 5642, 5528] dup_rows_per_dst=[2576, 2640, 2538, 2712, 2582, 2702, 2620, 2608] combine_rows_per_src=[5636, 5542, 5542, 5550, 5544, 5574, 5642, 5528] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279709696 dispatch_meta_bytes=229320 combine_payload_bytes=279709696 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279939016 ingress_bytes=499910656 link_bytes=499910656 token_bytes=559648712 weight_movement_bytes=220200960 total_nvlink_bytes=779849672 weight_to_token_ratio=0.393463 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694268520e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44370 payload_rows_remote=38724 dup_rows=21166 dup_rows_remote=18314 dedup_ratio=0.322968 router_tokens_per_expert=[520, 526, 533, 487, 495, 491, 514, 524, 483, 519, 536, 496, 511, 517, 481, 529, 533, 488, 506, 497, 497, 544, 505, 506, 505, 516, 529, 487, 514, 518, 529, 536, 524, 477, 546, 525, 530, 468, 484, 492, 473, 504, 493, 499, 511, 502, 506, 516, 530, 515, 517, 538, 495, 538, 542, 474, 514, 542, 517, 528, 514, 518, 543, 521] payload_rows_per_dst=[5646, 5584, 5624, 5446, 5484, 5520, 5494, 5572] dup_rows_per_dst=[2852, 2564, 2948, 2510, 2630, 2628, 2482, 2552] combine_rows_per_src=[5646, 5584, 5624, 5446, 5484, 5520, 5494, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277946368 dispatch_meta_bytes=228720 combine_payload_bytes=277946368 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278175088 ingress_bytes=718348288 link_bytes=718348288 token_bytes=556121456 weight_movement_bytes=440401920 total_nvlink_bytes=996523376 weight_to_token_ratio=0.791917 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.249325821e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278161408 dispatch_meta_bytes=229120 combine_payload_bytes=278161408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278390528 ingress_bytes=388261888 link_bytes=388261888 token_bytes=556551936 weight_movement_bytes=110100480 total_nvlink_bytes=666652416 weight_to_token_ratio=0.197826 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.752512813e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277602304 dispatch_meta_bytes=228768 combine_payload_bytes=277602304 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=277831072 ingress_bytes=828104704 link_bytes=828104704 token_bytes=555433376 weight_movement_bytes=550502400 total_nvlink_bytes=1105935776 weight_to_token_ratio=0.991122 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.440210280e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281028608 dispatch_meta_bytes=230160 combine_payload_bytes=281028608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281258768 ingress_bytes=391129088 link_bytes=391129088 token_bytes=562287376 weight_movement_bytes=110100480 total_nvlink_bytes=672387856 weight_to_token_ratio=0.195808 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.802378137e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=38968 dup_rows=21036 dup_rows_remote=18404 dedup_ratio=0.320984 router_tokens_per_expert=[491, 500, 502, 498, 481, 526, 486, 543, 517, 461, 533, 538, 520, 476, 525, 537, 535, 502, 493, 523, 509, 510, 534, 508, 478, 526, 492, 471, 527, 512, 512, 520, 513, 522, 502, 498, 529, 524, 487, 507, 509, 529, 535, 512, 531, 496, 516, 492, 479, 536, 493, 533, 497, 494, 524, 521, 526, 519, 525, 533, 510, 535, 517, 538] payload_rows_per_dst=[5520, 5536, 5594, 5526, 5532, 5562, 5588, 5642] dup_rows_per_dst=[2534, 2678, 2634, 2550, 2632, 2678, 2566, 2764] combine_rows_per_src=[5520, 5536, 5594, 5526, 5532, 5562, 5588, 5642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=229416 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279896104 ingress_bytes=389767168 link_bytes=389767168 token_bytes=559562792 weight_movement_bytes=110100480 total_nvlink_bytes=669663272 weight_to_token_ratio=0.196762 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778692108e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279365632 dispatch_meta_bytes=229872 combine_payload_bytes=279365632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279595504 ingress_bytes=389466112 link_bytes=389466112 token_bytes=558961136 weight_movement_bytes=110100480 total_nvlink_bytes=669061616 weight_to_token_ratio=0.196973 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773456249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277573632 dispatch_meta_bytes=228152 combine_payload_bytes=277573632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277801784 ingress_bytes=387674112 link_bytes=387674112 token_bytes=555375416 weight_movement_bytes=110100480 total_nvlink_bytes=665475896 weight_to_token_ratio=0.198245 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.742290422e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=39008 dup_rows=20946 dup_rows_remote=18454 dedup_ratio=0.319611 router_tokens_per_expert=[507, 533, 512, 533, 510, 495, 517, 505, 477, 467, 545, 488, 521, 508, 524, 507, 495, 514, 511, 529, 496, 493, 517, 484, 521, 508, 498, 497, 533, 550, 484, 564, 503, 502, 520, 528, 516, 530, 510, 540, 498, 484, 524, 549, 520, 501, 496, 499, 514, 529, 518, 523, 492, 482, 470, 509, 528, 498, 535, 508, 522, 536, 516, 525] payload_rows_per_dst=[5598, 5582, 5548, 5628, 5570, 5478, 5522, 5664] dup_rows_per_dst=[2626, 2492, 2530, 2682, 2728, 2664, 2552, 2672] combine_rows_per_src=[5598, 5582, 5548, 5628, 5570, 5478, 5522, 5664] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229488 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279552112 ingress_bytes=609624064 link_bytes=609624064 token_bytes=558874736 weight_movement_bytes=330301440 total_nvlink_bytes=889176176 weight_to_token_ratio=0.591012 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060236513e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44304 payload_rows_remote=38798 dup_rows=21232 dup_rows_remote=18650 dedup_ratio=0.323975 router_tokens_per_expert=[477, 562, 495, 474, 509, 519, 497, 501, 509, 472, 535, 513, 501, 546, 523, 506, 460, 536, 526, 519, 513, 501, 514, 541, 517, 445, 504, 505, 545, 470, 535, 523, 537, 511, 514, 496, 508, 519, 478, 482, 553, 492, 527, 518, 547, 527, 514, 503, 517, 496, 539, 481, 500, 492, 524, 514, 560, 544, 525, 498, 524, 509, 515, 511] payload_rows_per_dst=[5438, 5532, 5524, 5506, 5524, 5574, 5542, 5664] dup_rows_per_dst=[2630, 2678, 2696, 2582, 2566, 2788, 2584, 2708] combine_rows_per_src=[5438, 5532, 5524, 5506, 5524, 5574, 5542, 5664] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44362 payload_rows_remote=38764 dup_rows=21174 dup_rows_remote=18572 dedup_ratio=0.323090 router_tokens_per_expert=[573, 479, 509, 525, 532, 478, 543, 513, 496, 498, 543, 522, 520, 491, 510, 501, 534, 498, 506, 524, 491, 535, 507, 497, 540, 545, 522, 481, 510, 488, 465, 516, 556, 512, 492, 528, 497, 538, 514, 517, 510, 539, 470, 504, 524, 517, 512, 524, 509, 491, 525, 489, 534, 537, 497, 501, 527, 517, 508, 478, 525, 499, 490, 495] payload_rows_per_dst=[5554, 5532, 5524, 5534, 5582, 5598, 5518, 5520] dup_rows_per_dst=[2750, 2630, 2660, 2600, 2726, 2602, 2648, 2558] combine_rows_per_src=[5554, 5532, 5524, 5534, 5582, 5598, 5518, 5520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44574 payload_rows_remote=38990 dup_rows=20962 dup_rows_remote=18346 dedup_ratio=0.319855 router_tokens_per_expert=[492, 504, 520, 487, 511, 514, 515, 515, 521, 559, 499, 527, 503, 499, 512, 499, 474, 516, 501, 531, 491, 522, 535, 547, 531, 497, 507, 523, 504, 489, 535, 505, 541, 491, 518, 533, 495, 484, 526, 546, 548, 521, 495, 476, 489, 515, 511, 496, 539, 552, 523, 534, 478, 502, 481, 491, 522, 545, 478, 495, 525, 540, 507, 486] payload_rows_per_dst=[5482, 5576, 5542, 5570, 5618, 5600, 5584, 5602] dup_rows_per_dst=[2634, 2662, 2692, 2612, 2650, 2502, 2616, 2594] combine_rows_per_src=[5482, 5576, 5542, 5570, 5618, 5600, 5584, 5602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38912 dup_rows=21100 dup_rows_remote=18522 dedup_ratio=0.321960 router_tokens_per_expert=[519, 527, 522, 547, 505, 537, 456, 518, 516, 533, 505, 490, 503, 504, 519, 528, 529, 489, 503, 512, 537, 492, 525, 493, 524, 522, 506, 503, 500, 540, 504, 541, 484, 476, 544, 546, 520, 529, 532, 506, 501, 516, 528, 475, 508, 547, 497, 521, 497, 519, 485, 533, 507, 503, 499, 511, 534, 489, 538, 494, 467, 522, 497, 494] payload_rows_per_dst=[5634, 5514, 5532, 5602, 5554, 5556, 5520, 5524] dup_rows_per_dst=[2628, 2682, 2628, 2678, 2720, 2598, 2588, 2578] combine_rows_per_src=[5634, 5514, 5532, 5602, 5554, 5556, 5520, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=38968 dup_rows=21036 dup_rows_remote=18404 dedup_ratio=0.320984 router_tokens_per_expert=[491, 500, 502, 498, 481, 526, 486, 543, 517, 461, 533, 538, 520, 476, 525, 537, 535, 502, 493, 523, 509, 510, 534, 508, 478, 526, 492, 471, 527, 512, 512, 520, 513, 522, 502, 498, 529, 524, 487, 507, 509, 529, 535, 512, 531, 496, 516, 492, 479, 536, 493, 533, 497, 494, 524, 521, 526, 519, 525, 533, 510, 535, 517, 538] payload_rows_per_dst=[5520, 5536, 5594, 5526, 5532, 5562, 5588, 5642] dup_rows_per_dst=[2534, 2678, 2634, 2550, 2632, 2678, 2566, 2764] combine_rows_per_src=[5520, 5536, 5594, 5526, 5532, 5562, 5588, 5642] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279609344 dispatch_meta_bytes=229848 combine_payload_bytes=279609344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279839192 ingress_bytes=389709824 link_bytes=389709824 token_bytes=559448536 weight_movement_bytes=110100480 total_nvlink_bytes=669549016 weight_to_token_ratio=0.196802 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777694802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44772 payload_rows_remote=39206 dup_rows=20764 dup_rows_remote=18334 dedup_ratio=0.316833 router_tokens_per_expert=[488, 492, 529, 527, 550, 495, 470, 495, 502, 529, 490, 522, 488, 527, 505, 488, 475, 497, 490, 504, 501, 518, 520, 493, 531, 514, 543, 511, 510, 497, 492, 512, 528, 497, 506, 584, 505, 542, 530, 517, 540, 500, 492, 494, 537, 507, 538, 518, 511, 534, 545, 515, 532, 529, 498, 485, 521, 516, 509, 482, 474, 510, 515, 552] payload_rows_per_dst=[5544, 5554, 5566, 5592, 5670, 5584, 5648, 5614] dup_rows_per_dst=[2548, 2548, 2430, 2628, 2748, 2668, 2650, 2544] combine_rows_per_src=[5544, 5554, 5566, 5592, 5670, 5584, 5648, 5614] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44470 payload_rows_remote=38918 dup_rows=21066 dup_rows_remote=18528 dedup_ratio=0.321442 router_tokens_per_expert=[523, 537, 483, 480, 506, 505, 557, 497, 505, 509, 528, 504, 516, 550, 527, 501, 483, 465, 522, 530, 519, 501, 522, 503, 522, 527, 553, 509, 539, 497, 500, 543, 495, 517, 487, 476, 497, 542, 522, 497, 484, 511, 510, 515, 472, 540, 523, 516, 496, 514, 555, 521, 534, 527, 480, 500, 510, 503, 506, 537, 522, 500, 486, 510] payload_rows_per_dst=[5520, 5710, 5552, 5616, 5504, 5504, 5546, 5518] dup_rows_per_dst=[2656, 2570, 2538, 2764, 2562, 2638, 2708, 2630] combine_rows_per_src=[5520, 5710, 5552, 5616, 5504, 5504, 5546, 5518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38912 dup_rows=21100 dup_rows_remote=18522 dedup_ratio=0.321960 router_tokens_per_expert=[519, 527, 522, 547, 505, 537, 456, 518, 516, 533, 505, 490, 503, 504, 519, 528, 529, 489, 503, 512, 537, 492, 525, 493, 524, 522, 506, 503, 500, 540, 504, 541, 484, 476, 544, 546, 520, 529, 532, 506, 501, 516, 528, 475, 508, 547, 497, 521, 497, 519, 485, 533, 507, 503, 499, 511, 534, 489, 538, 494, 467, 522, 497, 494] payload_rows_per_dst=[5634, 5514, 5532, 5602, 5554, 5556, 5520, 5524] dup_rows_per_dst=[2628, 2682, 2628, 2678, 2720, 2598, 2588, 2578] combine_rows_per_src=[5634, 5514, 5532, 5602, 5554, 5556, 5520, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278104064 dispatch_meta_bytes=229792 combine_payload_bytes=278104064 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278333856 ingress_bytes=498305024 link_bytes=498305024 token_bytes=556437920 weight_movement_bytes=220200960 total_nvlink_bytes=776638880 weight_to_token_ratio=0.395733 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.666343938e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277860352 dispatch_meta_bytes=229344 combine_payload_bytes=277860352 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278089696 ingress_bytes=608161792 link_bytes=608161792 token_bytes=555950048 weight_movement_bytes=330301440 total_nvlink_bytes=886251488 weight_to_token_ratio=0.594121 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057693382e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=229344 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279709664 ingress_bytes=389580800 link_bytes=389580800 token_bytes=559189984 weight_movement_bytes=110100480 total_nvlink_bytes=669290464 weight_to_token_ratio=0.196893 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775450862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44612 payload_rows_remote=39036 dup_rows=20924 dup_rows_remote=18168 dedup_ratio=0.319275 router_tokens_per_expert=[527, 519, 517, 482, 489, 506, 517, 552, 518, 493, 525, 513, 519, 497, 493, 518, 483, 503, 515, 521, 489, 504, 505, 474, 527, 497, 525, 547, 525, 517, 523, 515, 515, 536, 521, 510, 511, 537, 476, 522, 517, 524, 543, 491, 479, 518, 523, 514, 493, 517, 543, 516, 472, 528, 525, 519, 504, 475, 514, 489, 501, 522, 533, 525] payload_rows_per_dst=[5576, 5578, 5662, 5526, 5486, 5574, 5724, 5486] dup_rows_per_dst=[2756, 2584, 2680, 2560, 2582, 2620, 2730, 2412] combine_rows_per_src=[5576, 5578, 5662, 5526, 5486, 5574, 5724, 5486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=229736 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279150952 ingress_bytes=499122176 link_bytes=499122176 token_bytes=558072168 weight_movement_bytes=220200960 total_nvlink_bytes=778273128 weight_to_token_ratio=0.394574 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680555556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=229784 combine_payload_bytes=278964224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279194008 ingress_bytes=389064704 link_bytes=389064704 token_bytes=558158232 weight_movement_bytes=110100480 total_nvlink_bytes=668258712 weight_to_token_ratio=0.197257 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766475104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44578 payload_rows_remote=39052 dup_rows=20958 dup_rows_remote=18378 dedup_ratio=0.319794 router_tokens_per_expert=[552, 521, 540, 552, 506, 498, 512, 497, 537, 472, 518, 487, 497, 526, 522, 519, 527, 524, 501, 524, 516, 501, 515, 513, 523, 470, 484, 532, 516, 524, 546, 545, 502, 511, 498, 519, 519, 494, 518, 492, 505, 493, 531, 531, 490, 503, 520, 499, 516, 500, 528, 527, 512, 537, 447, 484, 524, 520, 498, 470, 483, 572, 501, 507] payload_rows_per_dst=[5668, 5548, 5630, 5692, 5526, 5512, 5464, 5538] dup_rows_per_dst=[2688, 2608, 2612, 2588, 2580, 2632, 2638, 2612] combine_rows_per_src=[5668, 5548, 5630, 5692, 5526, 5512, 5464, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=229344 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279709664 ingress_bytes=389580800 link_bytes=389580800 token_bytes=559189984 weight_movement_bytes=110100480 total_nvlink_bytes=669290464 weight_to_token_ratio=0.196893 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775450862e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44362 payload_rows_remote=38764 dup_rows=21174 dup_rows_remote=18572 dedup_ratio=0.323090 router_tokens_per_expert=[573, 479, 509, 525, 532, 478, 543, 513, 496, 498, 543, 522, 520, 491, 510, 501, 534, 498, 506, 524, 491, 535, 507, 497, 540, 545, 522, 481, 510, 488, 465, 516, 556, 512, 492, 528, 497, 538, 514, 517, 510, 539, 470, 504, 524, 517, 512, 524, 509, 491, 525, 489, 534, 537, 497, 501, 527, 517, 508, 478, 525, 499, 490, 495] payload_rows_per_dst=[5554, 5532, 5524, 5534, 5582, 5598, 5518, 5520] dup_rows_per_dst=[2750, 2630, 2660, 2600, 2726, 2602, 2648, 2558] combine_rows_per_src=[5554, 5532, 5524, 5534, 5582, 5598, 5518, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=39008 dup_rows=20946 dup_rows_remote=18454 dedup_ratio=0.319611 router_tokens_per_expert=[507, 533, 512, 533, 510, 495, 517, 505, 477, 467, 545, 488, 521, 508, 524, 507, 495, 514, 511, 529, 496, 493, 517, 484, 521, 508, 498, 497, 533, 550, 484, 564, 503, 502, 520, 528, 516, 530, 510, 540, 498, 484, 524, 549, 520, 501, 496, 499, 514, 529, 518, 523, 492, 482, 470, 509, 528, 498, 535, 508, 522, 536, 516, 525] payload_rows_per_dst=[5598, 5582, 5548, 5628, 5570, 5478, 5522, 5664] dup_rows_per_dst=[2626, 2492, 2530, 2682, 2728, 2664, 2552, 2672] combine_rows_per_src=[5598, 5582, 5548, 5628, 5570, 5478, 5522, 5664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279322624 dispatch_meta_bytes=229488 combine_payload_bytes=279322624 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279552112 ingress_bytes=609624064 link_bytes=609624064 token_bytes=558874736 weight_movement_bytes=330301440 total_nvlink_bytes=889176176 weight_to_token_ratio=0.591012 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060236513e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281028608 dispatch_meta_bytes=230160 combine_payload_bytes=281028608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281258768 ingress_bytes=391129088 link_bytes=391129088 token_bytes=562287376 weight_movement_bytes=110100480 total_nvlink_bytes=672387856 weight_to_token_ratio=0.195808 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.802378137e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=229736 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279150952 ingress_bytes=499122176 link_bytes=499122176 token_bytes=558072168 weight_movement_bytes=220200960 total_nvlink_bytes=778273128 weight_to_token_ratio=0.394574 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.680555556e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279810048 dispatch_meta_bytes=228816 combine_payload_bytes=279810048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280038864 ingress_bytes=389910528 link_bytes=389910528 token_bytes=559848912 weight_movement_bytes=110100480 total_nvlink_bytes=669949392 weight_to_token_ratio=0.196661 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.781185375e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44382 payload_rows_remote=38854 dup_rows=21154 dup_rows_remote=18552 dedup_ratio=0.322784 router_tokens_per_expert=[493, 520, 542, 510, 495, 524, 469, 497, 495, 537, 506, 493, 503, 526, 512, 493, 508, 470, 539, 529, 526, 531, 536, 494, 526, 519, 494, 497, 540, 502, 543, 503, 508, 512, 527, 521, 541, 499, 526, 509, 540, 524, 521, 525, 454, 472, 506, 485, 520, 524, 510, 518, 501, 511, 531, 521, 509, 478, 545, 503, 501, 522, 499, 533] payload_rows_per_dst=[5462, 5528, 5626, 5542, 5590, 5532, 5556, 5546] dup_rows_per_dst=[2638, 2602, 2640, 2706, 2696, 2522, 2716, 2634] combine_rows_per_src=[5462, 5528, 5626, 5542, 5590, 5532, 5556, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279924736 dispatch_meta_bytes=229720 combine_payload_bytes=279924736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280154456 ingress_bytes=390025216 link_bytes=390025216 token_bytes=560079192 weight_movement_bytes=110100480 total_nvlink_bytes=670179672 weight_to_token_ratio=0.196580 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.783179987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277860352 dispatch_meta_bytes=229344 combine_payload_bytes=277860352 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278089696 ingress_bytes=608161792 link_bytes=608161792 token_bytes=555950048 weight_movement_bytes=330301440 total_nvlink_bytes=886251488 weight_to_token_ratio=0.594121 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057693382e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279609344 dispatch_meta_bytes=229848 combine_payload_bytes=279609344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279839192 ingress_bytes=389709824 link_bytes=389709824 token_bytes=559448536 weight_movement_bytes=110100480 total_nvlink_bytes=669549016 weight_to_token_ratio=0.196802 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.777694802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44394 payload_rows_remote=38838 dup_rows=21142 dup_rows_remote=18604 dedup_ratio=0.322601 router_tokens_per_expert=[515, 514, 530, 481, 541, 530, 528, 525, 527, 509, 499, 497, 546, 489, 481, 527, 508, 500, 481, 512, 521, 489, 498, 490, 494, 477, 536, 516, 507, 532, 462, 523, 498, 534, 506, 539, 470, 482, 507, 494, 511, 545, 531, 509, 483, 531, 480, 498, 547, 487, 525, 529, 503, 508, 542, 505, 523, 529, 526, 500, 535, 533, 520, 553] payload_rows_per_dst=[5696, 5490, 5498, 5556, 5476, 5522, 5566, 5590] dup_rows_per_dst=[2632, 2660, 2500, 2538, 2584, 2654, 2726, 2848] combine_rows_per_src=[5696, 5490, 5498, 5556, 5476, 5522, 5566, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44480 payload_rows_remote=38978 dup_rows=21056 dup_rows_remote=18552 dedup_ratio=0.321289 router_tokens_per_expert=[506, 494, 492, 518, 540, 537, 488, 541, 515, 524, 524, 486, 504, 514, 512, 523, 544, 513, 532, 478, 495, 522, 503, 501, 514, 507, 537, 526, 496, 514, 499, 513, 509, 520, 545, 513, 544, 519, 511, 510, 468, 543, 505, 484, 505, 496, 488, 514, 517, 495, 513, 503, 522, 483, 513, 503, 522, 491, 485, 519, 538, 519, 541, 518] payload_rows_per_dst=[5628, 5560, 5514, 5568, 5598, 5502, 5536, 5574] dup_rows_per_dst=[2604, 2644, 2662, 2644, 2744, 2504, 2562, 2692] combine_rows_per_src=[5628, 5560, 5514, 5568, 5598, 5502, 5536, 5574] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44446 payload_rows_remote=38768 dup_rows=21090 dup_rows_remote=18388 dedup_ratio=0.321808 router_tokens_per_expert=[531, 505, 502, 501, 481, 512, 493, 514, 479, 489, 489, 507, 501, 551, 521, 533, 520, 483, 491, 465, 516, 533, 522, 503, 506, 510, 527, 532, 527, 512, 527, 503, 534, 529, 520, 472, 479, 512, 511, 494, 533, 477, 521, 520, 517, 473, 545, 522, 503, 516, 567, 518, 529, 536, 516, 505, 512, 481, 536, 499, 550, 533, 504, 518] payload_rows_per_dst=[5526, 5520, 5514, 5578, 5526, 5540, 5678, 5564] dup_rows_per_dst=[2552, 2620, 2552, 2710, 2576, 2676, 2702, 2702] combine_rows_per_src=[5526, 5520, 5514, 5578, 5526, 5540, 5678, 5564] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278505472 dispatch_meta_bytes=229624 combine_payload_bytes=278505472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278735096 ingress_bytes=388605952 link_bytes=388605952 token_bytes=557240568 weight_movement_bytes=110100480 total_nvlink_bytes=667341048 weight_to_token_ratio=0.197582 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758496652e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38958 dup_rows=21034 dup_rows_remote=18396 dedup_ratio=0.320953 router_tokens_per_expert=[531, 510, 527, 502, 527, 472, 528, 519, 528, 541, 518, 523, 512, 491, 496, 473, 539, 498, 478, 562, 494, 508, 499, 522, 477, 540, 485, 520, 522, 501, 515, 494, 525, 525, 488, 493, 495, 507, 512, 523, 507, 504, 528, 507, 519, 501, 531, 513, 538, 542, 520, 524, 493, 485, 525, 527, 481, 496, 535, 538, 521, 488, 504, 521] payload_rows_per_dst=[5602, 5588, 5586, 5496, 5494, 5580, 5612, 5544] dup_rows_per_dst=[2492, 2714, 2614, 2586, 2654, 2640, 2696, 2638] combine_rows_per_src=[5602, 5588, 5586, 5496, 5494, 5580, 5612, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44578 payload_rows_remote=39052 dup_rows=20958 dup_rows_remote=18378 dedup_ratio=0.319794 router_tokens_per_expert=[552, 521, 540, 552, 506, 498, 512, 497, 537, 472, 518, 487, 497, 526, 522, 519, 527, 524, 501, 524, 516, 501, 515, 513, 523, 470, 484, 532, 516, 524, 546, 545, 502, 511, 498, 519, 519, 494, 518, 492, 505, 493, 531, 531, 490, 503, 520, 499, 516, 500, 528, 527, 512, 537, 447, 484, 524, 520, 498, 470, 483, 572, 501, 507] payload_rows_per_dst=[5668, 5548, 5630, 5692, 5526, 5512, 5464, 5538] dup_rows_per_dst=[2688, 2608, 2612, 2588, 2580, 2632, 2638, 2612] combine_rows_per_src=[5668, 5548, 5630, 5692, 5526, 5512, 5464, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44470 payload_rows_remote=38918 dup_rows=21066 dup_rows_remote=18528 dedup_ratio=0.321442 router_tokens_per_expert=[523, 537, 483, 480, 506, 505, 557, 497, 505, 509, 528, 504, 516, 550, 527, 501, 483, 465, 522, 530, 519, 501, 522, 503, 522, 527, 553, 509, 539, 497, 500, 543, 495, 517, 487, 476, 497, 542, 522, 497, 484, 511, 510, 515, 472, 540, 523, 516, 496, 514, 555, 521, 534, 527, 480, 500, 510, 503, 506, 537, 522, 500, 486, 510] payload_rows_per_dst=[5520, 5710, 5552, 5616, 5504, 5504, 5546, 5518] dup_rows_per_dst=[2656, 2570, 2538, 2764, 2562, 2638, 2708, 2630] combine_rows_per_src=[5520, 5710, 5552, 5616, 5504, 5504, 5546, 5518] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44308 payload_rows_remote=38760 dup_rows=21228 dup_rows_remote=18612 dedup_ratio=0.323914 router_tokens_per_expert=[482, 498, 513, 516, 524, 491, 510, 507, 474, 542, 519, 502, 525, 515, 501, 547, 514, 499, 540, 482, 522, 528, 496, 501, 491, 528, 535, 507, 524, 486, 508, 518, 521, 545, 513, 517, 547, 533, 519, 514, 481, 497, 523, 527, 504, 505, 499, 487, 505, 456, 530, 503, 514, 562, 509, 497, 507, 518, 541, 514, 472, 505, 529, 529] payload_rows_per_dst=[5434, 5592, 5548, 5574, 5632, 5452, 5512, 5564] dup_rows_per_dst=[2648, 2658, 2616, 2620, 2786, 2594, 2640, 2666] combine_rows_per_src=[5434, 5592, 5548, 5574, 5632, 5452, 5512, 5564] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278390784 dispatch_meta_bytes=229768 combine_payload_bytes=278390784 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278620552 ingress_bytes=608692224 link_bytes=608692224 token_bytes=557011336 weight_movement_bytes=330301440 total_nvlink_bytes=887312776 weight_to_token_ratio=0.592989 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058615890e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=230120 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279624424 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559018728 weight_movement_bytes=110100480 total_nvlink_bytes=669119208 weight_to_token_ratio=0.196953 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38958 dup_rows=21034 dup_rows_remote=18396 dedup_ratio=0.320953 router_tokens_per_expert=[531, 510, 527, 502, 527, 472, 528, 519, 528, 541, 518, 523, 512, 491, 496, 473, 539, 498, 478, 562, 494, 508, 499, 522, 477, 540, 485, 520, 522, 501, 515, 494, 525, 525, 488, 493, 495, 507, 512, 523, 507, 504, 528, 507, 519, 501, 531, 513, 538, 542, 520, 524, 493, 485, 525, 527, 481, 496, 535, 538, 521, 488, 504, 521] payload_rows_per_dst=[5602, 5588, 5586, 5496, 5494, 5580, 5612, 5544] dup_rows_per_dst=[2492, 2714, 2614, 2586, 2654, 2640, 2696, 2638] combine_rows_per_src=[5602, 5588, 5586, 5496, 5494, 5580, 5612, 5544] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44318 payload_rows_remote=38804 dup_rows=21218 dup_rows_remote=18466 dedup_ratio=0.323761 router_tokens_per_expert=[512, 523, 527, 498, 527, 505, 515, 502, 495, 509, 511, 514, 499, 516, 532, 526, 537, 498, 512, 493, 527, 478, 485, 508, 487, 539, 526, 477, 487, 554, 520, 478, 536, 511, 467, 468, 525, 503, 517, 510, 528, 522, 513, 544, 469, 545, 523, 532, 490, 557, 544, 522, 503, 528, 518, 503, 530, 500, 521, 479, 487, 551, 506, 499] payload_rows_per_dst=[5514, 5630, 5590, 5604, 5530, 5696, 5398, 5356] dup_rows_per_dst=[2752, 2574, 2834, 2534, 2544, 2978, 2534, 2468] combine_rows_per_src=[5514, 5630, 5590, 5604, 5530, 5696, 5398, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44480 payload_rows_remote=38978 dup_rows=21056 dup_rows_remote=18552 dedup_ratio=0.321289 router_tokens_per_expert=[506, 494, 492, 518, 540, 537, 488, 541, 515, 524, 524, 486, 504, 514, 512, 523, 544, 513, 532, 478, 495, 522, 503, 501, 514, 507, 537, 526, 496, 514, 499, 513, 509, 520, 545, 513, 544, 519, 511, 510, 468, 543, 505, 484, 505, 496, 488, 514, 517, 495, 513, 503, 522, 483, 513, 503, 522, 491, 485, 519, 538, 519, 541, 518] payload_rows_per_dst=[5628, 5560, 5514, 5568, 5598, 5502, 5536, 5574] dup_rows_per_dst=[2604, 2644, 2662, 2644, 2744, 2504, 2562, 2692] combine_rows_per_src=[5628, 5560, 5514, 5568, 5598, 5502, 5536, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277889024 dispatch_meta_bytes=228624 combine_payload_bytes=277889024 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278117648 ingress_bytes=608190464 link_bytes=608190464 token_bytes=556006672 weight_movement_bytes=330301440 total_nvlink_bytes=886308112 weight_to_token_ratio=0.594060 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057743247e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38974 dup_rows=21068 dup_rows_remote=18480 dedup_ratio=0.321472 router_tokens_per_expert=[507, 508, 519, 506, 512, 489, 507, 500, 508, 471, 518, 509, 512, 483, 499, 509, 477, 479, 554, 525, 520, 521, 551, 507, 527, 549, 497, 467, 548, 518, 499, 523, 506, 531, 473, 532, 498, 502, 510, 489, 503, 521, 521, 478, 522, 535, 491, 506, 472, 510, 505, 525, 527, 531, 519, 524, 468, 510, 542, 559, 548, 551, 514, 526] payload_rows_per_dst=[5500, 5488, 5594, 5650, 5494, 5554, 5608, 5580] dup_rows_per_dst=[2596, 2530, 2674, 2606, 2588, 2600, 2618, 2856] combine_rows_per_src=[5500, 5488, 5594, 5650, 5494, 5554, 5608, 5580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=229416 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279480360 ingress_bytes=499451904 link_bytes=499451904 token_bytes=558731304 weight_movement_bytes=220200960 total_nvlink_bytes=778932264 weight_to_token_ratio=0.394109 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686290068e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44382 payload_rows_remote=38854 dup_rows=21154 dup_rows_remote=18552 dedup_ratio=0.322784 router_tokens_per_expert=[493, 520, 542, 510, 495, 524, 469, 497, 495, 537, 506, 493, 503, 526, 512, 493, 508, 470, 539, 529, 526, 531, 536, 494, 526, 519, 494, 497, 540, 502, 543, 503, 508, 512, 527, 521, 541, 499, 526, 509, 540, 524, 521, 525, 454, 472, 506, 485, 520, 524, 510, 518, 501, 511, 531, 521, 509, 478, 545, 503, 501, 522, 499, 533] payload_rows_per_dst=[5462, 5528, 5626, 5542, 5590, 5532, 5556, 5546] dup_rows_per_dst=[2638, 2602, 2640, 2706, 2696, 2522, 2716, 2634] combine_rows_per_src=[5462, 5528, 5626, 5542, 5590, 5532, 5556, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279924736 dispatch_meta_bytes=229720 combine_payload_bytes=279924736 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280154456 ingress_bytes=390025216 link_bytes=390025216 token_bytes=560079192 weight_movement_bytes=110100480 total_nvlink_bytes=670179672 weight_to_token_ratio=0.196580 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.783179987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=229488 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278061168 ingress_bytes=387932160 link_bytes=387932160 token_bytes=555892848 weight_movement_bytes=110100480 total_nvlink_bytes=665993328 weight_to_token_ratio=0.198061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746778301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278964224 dispatch_meta_bytes=229784 combine_payload_bytes=278964224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279194008 ingress_bytes=389064704 link_bytes=389064704 token_bytes=558158232 weight_movement_bytes=110100480 total_nvlink_bytes=668258712 weight_to_token_ratio=0.197257 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766475104e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279250944 dispatch_meta_bytes=229416 combine_payload_bytes=279250944 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279480360 ingress_bytes=499451904 link_bytes=499451904 token_bytes=558731304 weight_movement_bytes=220200960 total_nvlink_bytes=778932264 weight_to_token_ratio=0.394109 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686290068e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278147072 dispatch_meta_bytes=229080 combine_payload_bytes=278147072 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278376152 ingress_bytes=498348032 link_bytes=498348032 token_bytes=556523224 weight_movement_bytes=220200960 total_nvlink_bytes=776724184 weight_to_token_ratio=0.395673 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.667091918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=39034 dup_rows=21014 dup_rows_remote=18370 dedup_ratio=0.320648 router_tokens_per_expert=[493, 506, 511, 505, 528, 522, 534, 510, 504, 486, 532, 484, 539, 500, 511, 510, 515, 522, 513, 497, 538, 494, 518, 492, 529, 524, 516, 566, 499, 487, 505, 484, 525, 517, 524, 540, 571, 514, 533, 512, 535, 522, 503, 454, 536, 510, 513, 486, 536, 481, 500, 485, 470, 557, 553, 483, 532, 514, 521, 496, 497, 511, 485, 478] payload_rows_per_dst=[5562, 5488, 5592, 5582, 5674, 5548, 5542, 5534] dup_rows_per_dst=[2656, 2644, 2586, 2638, 2798, 2570, 2588, 2534] combine_rows_per_src=[5562, 5488, 5592, 5582, 5674, 5548, 5542, 5534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279365632 dispatch_meta_bytes=229816 combine_payload_bytes=279365632 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279595448 ingress_bytes=499566592 link_bytes=499566592 token_bytes=558961080 weight_movement_bytes=220200960 total_nvlink_bytes=779162040 weight_to_token_ratio=0.393947 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688284681e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279394304 dispatch_meta_bytes=230120 combine_payload_bytes=279394304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279624424 ingress_bytes=389494784 link_bytes=389494784 token_bytes=559018728 weight_movement_bytes=110100480 total_nvlink_bytes=669119208 weight_to_token_ratio=0.196953 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.773954903e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278505472 dispatch_meta_bytes=229624 combine_payload_bytes=278505472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278735096 ingress_bytes=388605952 link_bytes=388605952 token_bytes=557240568 weight_movement_bytes=110100480 total_nvlink_bytes=667341048 weight_to_token_ratio=0.197582 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758496652e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44532 payload_rows_remote=39016 dup_rows=21004 dup_rows_remote=18394 dedup_ratio=0.320496 router_tokens_per_expert=[525, 508, 546, 499, 529, 539, 497, 494, 544, 545, 536, 500, 490, 516, 521, 497, 492, 500, 544, 511, 506, 476, 520, 510, 531, 506, 513, 473, 512, 514, 509, 505, 546, 534, 508, 530, 506, 497, 517, 510, 521, 505, 474, 552, 522, 534, 526, 501, 468, 497, 499, 530, 474, 521, 493, 506, 514, 522, 532, 553, 529, 503, 468, 468] payload_rows_per_dst=[5588, 5614, 5478, 5516, 5604, 5628, 5512, 5592] dup_rows_per_dst=[2686, 2684, 2640, 2610, 2692, 2642, 2464, 2586] combine_rows_per_src=[5588, 5614, 5478, 5516, 5604, 5628, 5512, 5592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44632 payload_rows_remote=39110 dup_rows=20904 dup_rows_remote=18322 dedup_ratio=0.318970 router_tokens_per_expert=[525, 485, 514, 537, 552, 476, 544, 523, 508, 515, 504, 518, 518, 523, 503, 512, 505, 511, 519, 488, 517, 449, 520, 509, 510, 491, 487, 547, 527, 503, 527, 484, 484, 548, 518, 503, 523, 519, 560, 520, 476, 509, 516, 491, 498, 517, 529, 516, 525, 510, 524, 469, 511, 516, 489, 512, 483, 522, 525, 532, 526, 528, 496, 522] payload_rows_per_dst=[5632, 5602, 5546, 5548, 5622, 5522, 5542, 5618] dup_rows_per_dst=[2680, 2600, 2490, 2604, 2728, 2582, 2570, 2650] combine_rows_per_src=[5632, 5602, 5546, 5548, 5622, 5522, 5542, 5618] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44290 payload_rows_remote=38760 dup_rows=21246 dup_rows_remote=18526 dedup_ratio=0.324188 router_tokens_per_expert=[493, 501, 536, 527, 516, 505, 490, 490, 492, 537, 518, 494, 517, 505, 513, 511, 490, 511, 521, 531, 559, 544, 490, 533, 475, 524, 521, 530, 479, 507, 526, 487, 493, 499, 525, 517, 533, 535, 501, 491, 497, 506, 506, 519, 479, 509, 525, 503, 537, 545, 504, 469, 470, 504, 563, 533, 500, 531, 509, 512, 539, 510, 511, 520] payload_rows_per_dst=[5488, 5546, 5586, 5456, 5596, 5544, 5530, 5544] dup_rows_per_dst=[2628, 2628, 2772, 2642, 2592, 2544, 2720, 2720] combine_rows_per_src=[5488, 5546, 5586, 5456, 5596, 5544, 5530, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229616 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280025328 ingress_bytes=389896192 link_bytes=389896192 token_bytes=559821040 weight_movement_bytes=110100480 total_nvlink_bytes=669921520 weight_to_token_ratio=0.196671 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780936048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44280 payload_rows_remote=38742 dup_rows=21256 dup_rows_remote=18428 dedup_ratio=0.324341 router_tokens_per_expert=[508, 557, 533, 591, 502, 525, 511, 528, 520, 480, 547, 472, 518, 515, 492, 550, 495, 528, 510, 560, 502, 537, 500, 519, 531, 506, 480, 490, 513, 511, 515, 528, 521, 474, 537, 523, 520, 524, 504, 491, 472, 502, 500, 516, 507, 517, 513, 543, 508, 476, 476, 483, 486, 490, 507, 476, 535, 477, 495, 528, 532, 520, 543, 498] payload_rows_per_dst=[5482, 5604, 5598, 5438, 5432, 5610, 5578, 5538] dup_rows_per_dst=[2446, 2584, 2768, 2600, 2496, 2790, 2744, 2828] combine_rows_per_src=[5482, 5604, 5598, 5438, 5432, 5610, 5578, 5538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38906 dup_rows=21118 dup_rows_remote=18586 dedup_ratio=0.322235 router_tokens_per_expert=[539, 522, 549, 490, 515, 464, 520, 495, 469, 513, 506, 518, 488, 543, 507, 549, 517, 506, 486, 496, 504, 508, 491, 514, 534, 505, 539, 536, 522, 549, 497, 516, 513, 519, 483, 546, 497, 458, 552, 525, 534, 518, 528, 505, 517, 526, 500, 509, 504, 512, 508, 511, 521, 563, 507, 503, 514, 514, 469, 519, 498, 474, 528, 486] payload_rows_per_dst=[5554, 5534, 5512, 5612, 5548, 5548, 5618, 5492] dup_rows_per_dst=[2634, 2652, 2532, 2784, 2638, 2726, 2640, 2512] combine_rows_per_src=[5554, 5534, 5512, 5612, 5548, 5548, 5618, 5492] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280340480 dispatch_meta_bytes=229728 combine_payload_bytes=280340480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280570208 ingress_bytes=390440960 link_bytes=390440960 token_bytes=560910688 weight_movement_bytes=110100480 total_nvlink_bytes=671011168 weight_to_token_ratio=0.196289 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790410459e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44290 payload_rows_remote=38760 dup_rows=21246 dup_rows_remote=18526 dedup_ratio=0.324188 router_tokens_per_expert=[493, 501, 536, 527, 516, 505, 490, 490, 492, 537, 518, 494, 517, 505, 513, 511, 490, 511, 521, 531, 559, 544, 490, 533, 475, 524, 521, 530, 479, 507, 526, 487, 493, 499, 525, 517, 533, 535, 501, 491, 497, 506, 506, 519, 479, 509, 525, 503, 537, 545, 504, 469, 470, 504, 563, 533, 500, 531, 509, 512, 539, 510, 511, 520] payload_rows_per_dst=[5488, 5546, 5586, 5456, 5596, 5544, 5530, 5544] dup_rows_per_dst=[2628, 2628, 2772, 2642, 2592, 2544, 2720, 2720] combine_rows_per_src=[5488, 5546, 5586, 5456, 5596, 5544, 5530, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44468 payload_rows_remote=38974 dup_rows=21068 dup_rows_remote=18480 dedup_ratio=0.321472 router_tokens_per_expert=[507, 508, 519, 506, 512, 489, 507, 500, 508, 471, 518, 509, 512, 483, 499, 509, 477, 479, 554, 525, 520, 521, 551, 507, 527, 549, 497, 467, 548, 518, 499, 523, 506, 531, 473, 532, 498, 502, 510, 489, 503, 521, 521, 478, 522, 535, 491, 506, 472, 510, 505, 525, 527, 531, 519, 524, 468, 510, 542, 559, 548, 551, 514, 526] payload_rows_per_dst=[5500, 5488, 5594, 5650, 5494, 5554, 5608, 5580] dup_rows_per_dst=[2596, 2530, 2674, 2606, 2588, 2600, 2618, 2856] combine_rows_per_src=[5500, 5488, 5594, 5650, 5494, 5554, 5608, 5580] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279666688 dispatch_meta_bytes=229640 combine_payload_bytes=279666688 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279896328 ingress_bytes=609968128 link_bytes=609968128 token_bytes=559563016 weight_movement_bytes=330301440 total_nvlink_bytes=889864456 weight_to_token_ratio=0.590285 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060834897e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44308 payload_rows_remote=38760 dup_rows=21228 dup_rows_remote=18612 dedup_ratio=0.323914 router_tokens_per_expert=[482, 498, 513, 516, 524, 491, 510, 507, 474, 542, 519, 502, 525, 515, 501, 547, 514, 499, 540, 482, 522, 528, 496, 501, 491, 528, 535, 507, 524, 486, 508, 518, 521, 545, 513, 517, 547, 533, 519, 514, 481, 497, 523, 527, 504, 505, 499, 487, 505, 456, 530, 503, 514, 562, 509, 497, 507, 518, 541, 514, 472, 505, 529, 529] payload_rows_per_dst=[5434, 5592, 5548, 5574, 5632, 5452, 5512, 5564] dup_rows_per_dst=[2648, 2658, 2616, 2620, 2786, 2594, 2640, 2666] combine_rows_per_src=[5434, 5592, 5548, 5574, 5632, 5452, 5512, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44280 payload_rows_remote=38742 dup_rows=21256 dup_rows_remote=18428 dedup_ratio=0.324341 router_tokens_per_expert=[508, 557, 533, 591, 502, 525, 511, 528, 520, 480, 547, 472, 518, 515, 492, 550, 495, 528, 510, 560, 502, 537, 500, 519, 531, 506, 480, 490, 513, 511, 515, 528, 521, 474, 537, 523, 520, 524, 504, 491, 472, 502, 500, 516, 507, 517, 513, 543, 508, 476, 476, 483, 486, 490, 507, 476, 535, 477, 495, 528, 532, 520, 543, 498] payload_rows_per_dst=[5482, 5604, 5598, 5438, 5432, 5610, 5578, 5538] dup_rows_per_dst=[2446, 2584, 2768, 2600, 2496, 2790, 2744, 2828] combine_rows_per_src=[5482, 5604, 5598, 5438, 5432, 5610, 5578, 5538] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38832 dup_rows=21088 dup_rows_remote=18300 dedup_ratio=0.321777 router_tokens_per_expert=[511, 525, 531, 540, 487, 527, 519, 556, 553, 525, 476, 484, 501, 500, 472, 468, 511, 527, 537, 529, 481, 508, 488, 503, 499, 547, 539, 517, 520, 524, 503, 481, 521, 494, 502, 522, 522, 477, 482, 498, 479, 535, 487, 550, 522, 521, 490, 480, 501, 546, 524, 530, 504, 481, 524, 521, 516, 542, 555, 493, 502, 498, 525, 535] payload_rows_per_dst=[5616, 5696, 5528, 5614, 5418, 5520, 5560, 5496] dup_rows_per_dst=[2788, 2642, 2640, 2636, 2616, 2608, 2702, 2456] combine_rows_per_src=[5616, 5696, 5528, 5614, 5418, 5520, 5560, 5496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44520 payload_rows_remote=38894 dup_rows=21016 dup_rows_remote=18452 dedup_ratio=0.320679 router_tokens_per_expert=[490, 539, 525, 541, 547, 488, 466, 545, 527, 507, 526, 478, 485, 503, 518, 511, 542, 515, 488, 495, 513, 525, 473, 541, 500, 484, 482, 533, 530, 539, 502, 447, 482, 500, 526, 502, 513, 512, 547, 513, 532, 535, 522, 514, 493, 512, 510, 540, 505, 529, 548, 485, 507, 551, 468, 520, 507, 529, 568, 490, 518, 499, 515, 471] payload_rows_per_dst=[5610, 5512, 5502, 5496, 5626, 5602, 5578, 5594] dup_rows_per_dst=[2672, 2598, 2682, 2538, 2564, 2714, 2648, 2600] combine_rows_per_src=[5610, 5512, 5502, 5496, 5626, 5602, 5578, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44632 payload_rows_remote=39110 dup_rows=20904 dup_rows_remote=18322 dedup_ratio=0.318970 router_tokens_per_expert=[525, 485, 514, 537, 552, 476, 544, 523, 508, 515, 504, 518, 518, 523, 503, 512, 505, 511, 519, 488, 517, 449, 520, 509, 510, 491, 487, 547, 527, 503, 527, 484, 484, 548, 518, 503, 523, 519, 560, 520, 476, 509, 516, 491, 498, 517, 529, 516, 525, 510, 524, 469, 511, 516, 489, 512, 483, 522, 525, 532, 526, 528, 496, 522] payload_rows_per_dst=[5632, 5602, 5546, 5548, 5622, 5522, 5542, 5618] dup_rows_per_dst=[2680, 2600, 2490, 2604, 2728, 2582, 2570, 2650] combine_rows_per_src=[5632, 5602, 5546, 5548, 5622, 5522, 5542, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=39034 dup_rows=21014 dup_rows_remote=18370 dedup_ratio=0.320648 router_tokens_per_expert=[493, 506, 511, 505, 528, 522, 534, 510, 504, 486, 532, 484, 539, 500, 511, 510, 515, 522, 513, 497, 538, 494, 518, 492, 529, 524, 516, 566, 499, 487, 505, 484, 525, 517, 524, 540, 571, 514, 533, 512, 535, 522, 503, 454, 536, 510, 513, 486, 536, 481, 500, 485, 470, 557, 553, 483, 532, 514, 521, 496, 497, 511, 485, 478] payload_rows_per_dst=[5562, 5488, 5592, 5582, 5674, 5548, 5542, 5534] dup_rows_per_dst=[2656, 2644, 2586, 2638, 2798, 2570, 2588, 2534] combine_rows_per_src=[5562, 5488, 5592, 5582, 5674, 5548, 5542, 5534] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=229144 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278060824 ingress_bytes=387932160 link_bytes=387932160 token_bytes=555892504 weight_movement_bytes=110100480 total_nvlink_bytes=665992984 weight_to_token_ratio=0.198061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746778301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277702656 dispatch_meta_bytes=228680 combine_payload_bytes=277702656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277931336 ingress_bytes=497903616 link_bytes=497903616 token_bytes=555633992 weight_movement_bytes=220200960 total_nvlink_bytes=775834952 weight_to_token_ratio=0.396306 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.659362793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279365632 dispatch_meta_bytes=229816 combine_payload_bytes=279365632 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279595448 ingress_bytes=499566592 link_bytes=499566592 token_bytes=558961080 weight_movement_bytes=220200960 total_nvlink_bytes=779162040 weight_to_token_ratio=0.393947 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.688284681e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=229968 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279108176 ingress_bytes=388978688 link_bytes=388978688 token_bytes=557986384 weight_movement_bytes=110100480 total_nvlink_bytes=668086864 weight_to_token_ratio=0.197318 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764979144e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277831680 dispatch_meta_bytes=229488 combine_payload_bytes=277831680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278061168 ingress_bytes=387932160 link_bytes=387932160 token_bytes=555892848 weight_movement_bytes=110100480 total_nvlink_bytes=665993328 weight_to_token_ratio=0.198061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746778301e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277702656 dispatch_meta_bytes=228680 combine_payload_bytes=277702656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277931336 ingress_bytes=497903616 link_bytes=497903616 token_bytes=555633992 weight_movement_bytes=220200960 total_nvlink_bytes=775834952 weight_to_token_ratio=0.396306 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.659362793e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278347776 dispatch_meta_bytes=228528 combine_payload_bytes=278347776 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278576304 ingress_bytes=608649216 link_bytes=608649216 token_bytes=556924080 weight_movement_bytes=330301440 total_nvlink_bytes=887225520 weight_to_token_ratio=0.593082 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.058541092e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=229384 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279021576 ingress_bytes=498993152 link_bytes=498993152 token_bytes=557813768 weight_movement_bytes=220200960 total_nvlink_bytes=778014728 weight_to_token_ratio=0.394757 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678311616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=39020 dup_rows=20940 dup_rows_remote=18318 dedup_ratio=0.319519 router_tokens_per_expert=[573, 515, 512, 525, 521, 524, 518, 530, 519, 492, 481, 481, 540, 515, 534, 537, 504, 513, 515, 501, 503, 507, 478, 518, 523, 476, 492, 509, 528, 493, 529, 496, 537, 482, 507, 544, 545, 525, 510, 520, 533, 497, 523, 528, 534, 513, 500, 491, 462, 510, 538, 507, 488, 481, 507, 488, 513, 536, 513, 498, 480, 507, 552, 497] payload_rows_per_dst=[5740, 5576, 5492, 5490, 5552, 5634, 5504, 5608] dup_rows_per_dst=[2696, 2622, 2586, 2602, 2788, 2604, 2458, 2584] combine_rows_per_src=[5740, 5576, 5492, 5490, 5552, 5634, 5504, 5608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38890 dup_rows=21130 dup_rows_remote=18468 dedup_ratio=0.322418 router_tokens_per_expert=[505, 523, 511, 503, 525, 518, 496, 536, 532, 496, 471, 494, 518, 519, 539, 531, 565, 480, 501, 504, 518, 509, 514, 507, 520, 509, 498, 484, 519, 535, 515, 532, 531, 506, 532, 505, 520, 537, 480, 500, 512, 546, 531, 514, 482, 518, 474, 512, 508, 514, 496, 537, 496, 509, 509, 526, 522, 489, 486, 508, 515, 488, 523, 515] payload_rows_per_dst=[5600, 5548, 5522, 5516, 5598, 5516, 5606, 5500] dup_rows_per_dst=[2634, 2652, 2674, 2708, 2624, 2662, 2584, 2592] combine_rows_per_src=[5600, 5548, 5522, 5516, 5598, 5516, 5606, 5500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229616 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280025328 ingress_bytes=389896192 link_bytes=389896192 token_bytes=559821040 weight_movement_bytes=110100480 total_nvlink_bytes=669921520 weight_to_token_ratio=0.196671 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780936048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280340480 dispatch_meta_bytes=229728 combine_payload_bytes=280340480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280570208 ingress_bytes=390440960 link_bytes=390440960 token_bytes=560910688 weight_movement_bytes=110100480 total_nvlink_bytes=671011168 weight_to_token_ratio=0.196289 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790410459e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44614 payload_rows_remote=39114 dup_rows=20922 dup_rows_remote=18256 dedup_ratio=0.319244 router_tokens_per_expert=[525, 514, 494, 476, 502, 527, 488, 521, 516, 512, 507, 499, 511, 522, 478, 508, 534, 531, 479, 580, 529, 475, 516, 507, 516, 502, 541, 521, 502, 486, 518, 497, 490, 510, 529, 559, 510, 517, 494, 503, 569, 498, 539, 510, 520, 501, 494, 473, 542, 519, 518, 508, 520, 524, 496, 499, 509, 493, 503, 535, 491, 512, 509, 540] payload_rows_per_dst=[5524, 5546, 5632, 5500, 5590, 5600, 5630, 5592] dup_rows_per_dst=[2570, 2560, 2670, 2666, 2634, 2608, 2622, 2592] combine_rows_per_src=[5524, 5546, 5632, 5500, 5590, 5600, 5630, 5592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44456 payload_rows_remote=38944 dup_rows=21080 dup_rows_remote=18462 dedup_ratio=0.321655 router_tokens_per_expert=[483, 551, 489, 488, 508, 473, 497, 526, 479, 518, 532, 576, 493, 514, 513, 480, 584, 485, 538, 548, 555, 529, 535, 490, 518, 500, 493, 508, 537, 530, 515, 527, 537, 525, 490, 511, 545, 506, 499, 500, 489, 477, 482, 518, 529, 505, 510, 503, 519, 485, 516, 529, 500, 495, 506, 515, 524, 486, 493, 527, 534, 485, 524, 492] payload_rows_per_dst=[5518, 5558, 5666, 5606, 5556, 5470, 5512, 5570] dup_rows_per_dst=[2512, 2652, 2862, 2650, 2670, 2556, 2618, 2560] combine_rows_per_src=[5518, 5558, 5666, 5606, 5556, 5470, 5512, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44520 payload_rows_remote=38894 dup_rows=21016 dup_rows_remote=18452 dedup_ratio=0.320679 router_tokens_per_expert=[490, 539, 525, 541, 547, 488, 466, 545, 527, 507, 526, 478, 485, 503, 518, 511, 542, 515, 488, 495, 513, 525, 473, 541, 500, 484, 482, 533, 530, 539, 502, 447, 482, 500, 526, 502, 513, 512, 547, 513, 532, 535, 522, 514, 493, 512, 510, 540, 505, 529, 548, 485, 507, 551, 468, 520, 507, 529, 568, 490, 518, 499, 515, 471] payload_rows_per_dst=[5610, 5512, 5502, 5496, 5626, 5602, 5578, 5594] dup_rows_per_dst=[2672, 2598, 2682, 2538, 2564, 2714, 2648, 2600] combine_rows_per_src=[5610, 5512, 5502, 5496, 5626, 5602, 5578, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280369152 dispatch_meta_bytes=229480 combine_payload_bytes=280369152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280598632 ingress_bytes=390469632 link_bytes=390469632 token_bytes=560967784 weight_movement_bytes=110100480 total_nvlink_bytes=671068264 weight_to_token_ratio=0.196269 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790909113e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44418 payload_rows_remote=38906 dup_rows=21118 dup_rows_remote=18586 dedup_ratio=0.322235 router_tokens_per_expert=[539, 522, 549, 490, 515, 464, 520, 495, 469, 513, 506, 518, 488, 543, 507, 549, 517, 506, 486, 496, 504, 508, 491, 514, 534, 505, 539, 536, 522, 549, 497, 516, 513, 519, 483, 546, 497, 458, 552, 525, 534, 518, 528, 505, 517, 526, 500, 509, 504, 512, 508, 511, 521, 563, 507, 503, 514, 514, 469, 519, 498, 474, 528, 486] payload_rows_per_dst=[5554, 5534, 5512, 5612, 5548, 5548, 5618, 5492] dup_rows_per_dst=[2634, 2652, 2532, 2784, 2638, 2726, 2640, 2512] combine_rows_per_src=[5554, 5534, 5512, 5612, 5548, 5548, 5618, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44456 payload_rows_remote=38790 dup_rows=21080 dup_rows_remote=18302 dedup_ratio=0.321655 router_tokens_per_expert=[491, 503, 535, 505, 479, 507, 483, 489, 518, 516, 519, 519, 540, 511, 528, 493, 507, 499, 534, 482, 485, 506, 553, 517, 497, 527, 529, 515, 508, 505, 517, 518, 512, 487, 542, 526, 502, 502, 493, 503, 523, 541, 523, 533, 523, 520, 520, 463, 502, 480, 498, 489, 554, 509, 480, 520, 550, 517, 502, 509, 552, 556, 497, 505] payload_rows_per_dst=[5472, 5440, 5550, 5520, 5358, 5656, 5794, 5666] dup_rows_per_dst=[2512, 2552, 2548, 2712, 2386, 2636, 2956, 2778] combine_rows_per_src=[5472, 5440, 5550, 5520, 5358, 5656, 5794, 5666] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279695360 dispatch_meta_bytes=229352 combine_payload_bytes=279695360 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279924712 ingress_bytes=499896320 link_bytes=499896320 token_bytes=559620072 weight_movement_bytes=220200960 total_nvlink_bytes=779821032 weight_to_token_ratio=0.393483 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694019193e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44654 payload_rows_remote=39058 dup_rows=20882 dup_rows_remote=18300 dedup_ratio=0.318634 router_tokens_per_expert=[518, 495, 493, 514, 535, 482, 514, 520, 528, 528, 510, 519, 496, 534, 513, 511, 538, 518, 495, 483, 490, 541, 521, 503, 510, 532, 521, 531, 506, 509, 516, 500, 529, 446, 549, 504, 538, 511, 503, 531, 533, 501, 512, 552, 506, 503, 513, 491, 514, 497, 497, 484, 504, 485, 465, 514, 526, 535, 557, 503, 555, 510, 469, 507] payload_rows_per_dst=[5502, 5640, 5596, 5608, 5602, 5618, 5464, 5624] dup_rows_per_dst=[2640, 2638, 2582, 2642, 2620, 2604, 2456, 2700] combine_rows_per_src=[5502, 5640, 5596, 5608, 5602, 5618, 5464, 5624] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229432 combine_payload_bytes=278763520 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278992952 ingress_bytes=609064960 link_bytes=609064960 token_bytes=557756472 weight_movement_bytes=330301440 total_nvlink_bytes=888057912 weight_to_token_ratio=0.592197 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059264139e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=39012 dup_rows=20948 dup_rows_remote=18190 dedup_ratio=0.319641 router_tokens_per_expert=[506, 514, 485, 523, 513, 510, 526, 500, 529, 514, 537, 532, 481, 518, 554, 520, 517, 498, 515, 464, 505, 465, 529, 521, 514, 520, 523, 517, 498, 536, 504, 525, 514, 512, 521, 510, 519, 537, 540, 531, 522, 511, 532, 515, 523, 511, 520, 496, 500, 543, 498, 499, 490, 494, 513, 516, 497, 501, 499, 501, 480, 530, 473, 507] payload_rows_per_dst=[5576, 5662, 5480, 5622, 5670, 5544, 5560, 5474] dup_rows_per_dst=[2758, 2780, 2492, 2660, 2698, 2512, 2546, 2502] combine_rows_per_src=[5576, 5662, 5480, 5622, 5670, 5544, 5560, 5474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=229624 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279380216 ingress_bytes=389251072 link_bytes=389251072 token_bytes=558530808 weight_movement_bytes=110100480 total_nvlink_bytes=668631288 weight_to_token_ratio=0.197125 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769716350e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38890 dup_rows=21130 dup_rows_remote=18468 dedup_ratio=0.322418 router_tokens_per_expert=[505, 523, 511, 503, 525, 518, 496, 536, 532, 496, 471, 494, 518, 519, 539, 531, 565, 480, 501, 504, 518, 509, 514, 507, 520, 509, 498, 484, 519, 535, 515, 532, 531, 506, 532, 505, 520, 537, 480, 500, 512, 546, 531, 514, 482, 518, 474, 512, 508, 514, 496, 537, 496, 509, 509, 526, 522, 489, 486, 508, 515, 488, 523, 515] payload_rows_per_dst=[5600, 5548, 5522, 5516, 5598, 5516, 5606, 5500] dup_rows_per_dst=[2634, 2652, 2674, 2708, 2624, 2662, 2584, 2592] combine_rows_per_src=[5600, 5548, 5522, 5516, 5598, 5516, 5606, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44596 payload_rows_remote=39020 dup_rows=20940 dup_rows_remote=18318 dedup_ratio=0.319519 router_tokens_per_expert=[573, 515, 512, 525, 521, 524, 518, 530, 519, 492, 481, 481, 540, 515, 534, 537, 504, 513, 515, 501, 503, 507, 478, 518, 523, 476, 492, 509, 528, 493, 529, 496, 537, 482, 507, 544, 545, 525, 510, 520, 533, 497, 523, 528, 534, 513, 500, 491, 462, 510, 538, 507, 488, 481, 507, 488, 513, 536, 513, 498, 480, 507, 552, 497] payload_rows_per_dst=[5740, 5576, 5492, 5490, 5552, 5634, 5504, 5608] dup_rows_per_dst=[2696, 2622, 2586, 2602, 2788, 2604, 2458, 2584] combine_rows_per_src=[5740, 5576, 5492, 5490, 5552, 5634, 5504, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278792192 dispatch_meta_bytes=229384 combine_payload_bytes=278792192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279021576 ingress_bytes=498993152 link_bytes=498993152 token_bytes=557813768 weight_movement_bytes=220200960 total_nvlink_bytes=778014728 weight_to_token_ratio=0.394757 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678311616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280369152 dispatch_meta_bytes=229480 combine_payload_bytes=280369152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280598632 ingress_bytes=390469632 link_bytes=390469632 token_bytes=560967784 weight_movement_bytes=110100480 total_nvlink_bytes=671068264 weight_to_token_ratio=0.196269 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.790909113e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38902 dup_rows=21124 dup_rows_remote=18524 dedup_ratio=0.322327 router_tokens_per_expert=[528, 504, 528, 509, 482, 520, 489, 524, 503, 567, 499, 460, 517, 498, 529, 486, 511, 510, 506, 524, 555, 545, 487, 527, 496, 511, 473, 502, 532, 506, 512, 512, 482, 501, 519, 506, 529, 497, 517, 504, 498, 527, 492, 507, 553, 544, 529, 509, 500, 514, 518, 505, 486, 531, 522, 531, 513, 528, 519, 530, 509, 517, 480, 499] payload_rows_per_dst=[5568, 5544, 5606, 5492, 5510, 5638, 5536, 5518] dup_rows_per_dst=[2600, 2574, 2724, 2596, 2600, 2680, 2678, 2672] combine_rows_per_src=[5568, 5544, 5606, 5492, 5510, 5638, 5536, 5518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279150592 dispatch_meta_bytes=229624 combine_payload_bytes=279150592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279380216 ingress_bytes=389251072 link_bytes=389251072 token_bytes=558530808 weight_movement_bytes=110100480 total_nvlink_bytes=668631288 weight_to_token_ratio=0.197125 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.769716350e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=229968 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279108176 ingress_bytes=388978688 link_bytes=388978688 token_bytes=557986384 weight_movement_bytes=110100480 total_nvlink_bytes=668086864 weight_to_token_ratio=0.197318 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764979144e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278046720 dispatch_meta_bytes=228368 combine_payload_bytes=278046720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278275088 ingress_bytes=388147200 link_bytes=388147200 token_bytes=556321808 weight_movement_bytes=110100480 total_nvlink_bytes=666422288 weight_to_token_ratio=0.197908 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750518200e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279967744 dispatch_meta_bytes=229432 combine_payload_bytes=279967744 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280197176 ingress_bytes=610269184 link_bytes=610269184 token_bytes=560164920 weight_movement_bytes=330301440 total_nvlink_bytes=890466360 weight_to_token_ratio=0.589650 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061358483e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279638016 dispatch_meta_bytes=228808 combine_payload_bytes=279638016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279866824 ingress_bytes=389738496 link_bytes=389738496 token_bytes=559504840 weight_movement_bytes=110100480 total_nvlink_bytes=669605320 weight_to_token_ratio=0.196782 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778193455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229704 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279079240 ingress_bytes=388950016 link_bytes=388950016 token_bytes=557928776 weight_movement_bytes=110100480 total_nvlink_bytes=668029256 weight_to_token_ratio=0.197338 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764480491e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=38962 dup_rows=21036 dup_rows_remote=18460 dedup_ratio=0.320984 router_tokens_per_expert=[518, 529, 509, 461, 507, 504, 486, 469, 515, 553, 491, 524, 498, 496, 490, 490, 505, 548, 499, 500, 496, 496, 528, 482, 558, 513, 503, 514, 546, 539, 523, 521, 551, 490, 526, 526, 501, 548, 530, 515, 489, 506, 559, 543, 519, 508, 483, 490, 509, 493, 523, 506, 514, 519, 529, 510, 517, 505, 510, 485, 501, 514, 548, 490] payload_rows_per_dst=[5518, 5538, 5508, 5624, 5692, 5562, 5554, 5504] dup_rows_per_dst=[2448, 2576, 2600, 2810, 2682, 2632, 2652, 2636] combine_rows_per_src=[5518, 5538, 5508, 5624, 5692, 5562, 5554, 5504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44556 payload_rows_remote=39010 dup_rows=20980 dup_rows_remote=18476 dedup_ratio=0.320129 router_tokens_per_expert=[542, 489, 535, 515, 485, 490, 484, 473, 544, 505, 489, 497, 503, 480, 535, 517, 534, 508, 516, 491, 515, 536, 508, 527, 531, 487, 530, 499, 500, 547, 503, 533, 566, 475, 478, 539, 526, 497, 515, 499, 506, 502, 507, 503, 499, 500, 515, 493, 534, 530, 532, 465, 530, 524, 509, 504, 527, 542, 528, 512, 505, 526, 495, 537] payload_rows_per_dst=[5556, 5542, 5546, 5604, 5552, 5546, 5572, 5638] dup_rows_per_dst=[2470, 2598, 2724, 2656, 2638, 2504, 2684, 2706] combine_rows_per_src=[5556, 5542, 5546, 5604, 5552, 5546, 5572, 5638] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5577164200 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=2789729704 ingress_bytes=4659142656 link_bytes=4659142656 total_nvlink_bytes=7448872360 weight_to_token_ratio=0.335602 prefetch_share_of_ingress=0.212680 dedup_ratio=0.321945 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.103015376e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278763520 dispatch_meta_bytes=229432 combine_payload_bytes=278763520 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=278992952 ingress_bytes=609064960 link_bytes=609064960 token_bytes=557756472 weight_movement_bytes=330301440 total_nvlink_bytes=888057912 weight_to_token_ratio=0.592197 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059264139e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=39000 dup_rows=20968 dup_rows_remote=18342 dedup_ratio=0.319946 router_tokens_per_expert=[538, 486, 489, 539, 493, 547, 532, 508, 518, 524, 515, 540, 531, 530, 528, 497, 529, 496, 522, 503, 480, 475, 524, 485, 526, 527, 487, 539, 491, 470, 539, 518, 523, 497, 508, 506, 520, 486, 495, 496, 542, 485, 504, 467, 494, 495, 536, 495, 564, 529, 548, 527, 500, 548, 503, 513, 494, 474, 489, 547, 510, 541, 505, 501] payload_rows_per_dst=[5668, 5668, 5450, 5568, 5510, 5510, 5700, 5494] dup_rows_per_dst=[2596, 2698, 2578, 2626, 2552, 2526, 2764, 2628] combine_rows_per_src=[5668, 5668, 5450, 5568, 5510, 5510, 5700, 5494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38924 dup_rows=21076 dup_rows_remote=18474 dedup_ratio=0.321594 router_tokens_per_expert=[477, 513, 496, 518, 514, 520, 483, 511, 508, 500, 529, 531, 543, 542, 517, 504, 524, 521, 470, 530, 555, 490, 521, 461, 519, 521, 506, 509, 513, 513, 539, 475, 508, 517, 492, 507, 488, 490, 563, 542, 567, 498, 537, 516, 524, 524, 520, 531, 494, 518, 474, 524, 507, 539, 501, 512, 543, 456, 509, 504, 492, 494, 506, 498] payload_rows_per_dst=[5514, 5640, 5530, 5546, 5556, 5668, 5536, 5470] dup_rows_per_dst=[2550, 2708, 2614, 2644, 2658, 2766, 2602, 2534] combine_rows_per_src=[5514, 5640, 5530, 5546, 5556, 5668, 5536, 5470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38764 dup_rows=21114 dup_rows_remote=18196 dedup_ratio=0.322174 router_tokens_per_expert=[537, 540, 508, 488, 521, 501, 492, 512, 512, 507, 535, 533, 525, 476, 525, 517, 496, 514, 521, 505, 472, 515, 510, 508, 546, 505, 495, 530, 486, 520, 510, 528, 518, 513, 520, 532, 501, 509, 523, 491, 494, 539, 505, 532, 521, 514, 502, 518, 515, 511, 517, 526, 506, 500, 486, 513, 527, 486, 492, 471, 574, 512, 478, 532] payload_rows_per_dst=[5572, 5358, 5772, 5410, 5530, 5534, 5588, 5658] dup_rows_per_dst=[2626, 2354, 2858, 2398, 2684, 2618, 2658, 2918] combine_rows_per_src=[5572, 5358, 5772, 5410, 5530, 5534, 5588, 5658] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=229688 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279509304 ingress_bytes=499480576 link_bytes=499480576 token_bytes=558788920 weight_movement_bytes=220200960 total_nvlink_bytes=778989880 weight_to_token_ratio=0.394068 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686788721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279695360 dispatch_meta_bytes=229352 combine_payload_bytes=279695360 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279924712 ingress_bytes=499896320 link_bytes=499896320 token_bytes=559620072 weight_movement_bytes=220200960 total_nvlink_bytes=779821032 weight_to_token_ratio=0.393483 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694019193e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=229944 combine_payload_bytes=279623680 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279853624 ingress_bytes=609925120 link_bytes=609925120 token_bytes=559477304 weight_movement_bytes=330301440 total_nvlink_bytes=889778744 weight_to_token_ratio=0.590375 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060760099e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44460 payload_rows_remote=38924 dup_rows=21076 dup_rows_remote=18474 dedup_ratio=0.321594 router_tokens_per_expert=[477, 513, 496, 518, 514, 520, 483, 511, 508, 500, 529, 531, 543, 542, 517, 504, 524, 521, 470, 530, 555, 490, 521, 461, 519, 521, 506, 509, 513, 513, 539, 475, 508, 517, 492, 507, 488, 490, 563, 542, 567, 498, 537, 516, 524, 524, 520, 531, 494, 518, 474, 524, 507, 539, 501, 512, 543, 456, 509, 504, 492, 494, 506, 498] payload_rows_per_dst=[5514, 5640, 5530, 5546, 5556, 5668, 5536, 5470] dup_rows_per_dst=[2550, 2708, 2614, 2644, 2658, 2766, 2602, 2534] combine_rows_per_src=[5514, 5640, 5530, 5546, 5556, 5668, 5536, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=229368 combine_payload_bytes=279552000 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279781368 ingress_bytes=609853440 link_bytes=609853440 token_bytes=559333368 weight_movement_bytes=330301440 total_nvlink_bytes=889634808 weight_to_token_ratio=0.590527 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060635436e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44412 payload_rows_remote=38902 dup_rows=21124 dup_rows_remote=18524 dedup_ratio=0.322327 router_tokens_per_expert=[528, 504, 528, 509, 482, 520, 489, 524, 503, 567, 499, 460, 517, 498, 529, 486, 511, 510, 506, 524, 555, 545, 487, 527, 496, 511, 473, 502, 532, 506, 512, 512, 482, 501, 519, 506, 529, 497, 517, 504, 498, 527, 492, 507, 553, 544, 529, 509, 500, 514, 518, 505, 486, 531, 522, 531, 513, 528, 519, 530, 509, 517, 480, 499] payload_rows_per_dst=[5568, 5544, 5606, 5492, 5510, 5638, 5536, 5518] dup_rows_per_dst=[2600, 2574, 2724, 2596, 2600, 2680, 2678, 2672] combine_rows_per_src=[5568, 5544, 5606, 5492, 5510, 5638, 5536, 5518] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44296 payload_rows_remote=38792 dup_rows=21240 dup_rows_remote=18714 dedup_ratio=0.324097 router_tokens_per_expert=[508, 532, 464, 515, 485, 509, 528, 518, 512, 528, 537, 567, 534, 523, 521, 487, 524, 515, 521, 501, 475, 488, 478, 513, 513, 511, 505, 516, 478, 550, 517, 534, 487, 484, 510, 515, 514, 517, 513, 484, 537, 514, 493, 503, 528, 478, 537, 553, 469, 539, 532, 496, 529, 517, 532, 522, 499, 534, 540, 514, 454, 473, 519, 525] payload_rows_per_dst=[5444, 5706, 5504, 5500, 5470, 5514, 5578, 5580] dup_rows_per_dst=[2674, 2712, 2526, 2748, 2578, 2772, 2694, 2536] combine_rows_per_src=[5444, 5706, 5504, 5500, 5470, 5514, 5578, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44654 payload_rows_remote=39058 dup_rows=20882 dup_rows_remote=18300 dedup_ratio=0.318634 router_tokens_per_expert=[518, 495, 493, 514, 535, 482, 514, 520, 528, 528, 510, 519, 496, 534, 513, 511, 538, 518, 495, 483, 490, 541, 521, 503, 510, 532, 521, 531, 506, 509, 516, 500, 529, 446, 549, 504, 538, 511, 503, 531, 533, 501, 512, 552, 506, 503, 513, 491, 514, 497, 497, 484, 504, 485, 465, 514, 526, 535, 557, 503, 555, 510, 469, 507] payload_rows_per_dst=[5502, 5640, 5596, 5608, 5602, 5618, 5464, 5624] dup_rows_per_dst=[2640, 2638, 2582, 2642, 2620, 2604, 2456, 2700] combine_rows_per_src=[5502, 5640, 5596, 5608, 5602, 5618, 5464, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38764 dup_rows=21114 dup_rows_remote=18196 dedup_ratio=0.322174 router_tokens_per_expert=[537, 540, 508, 488, 521, 501, 492, 512, 512, 507, 535, 533, 525, 476, 525, 517, 496, 514, 521, 505, 472, 515, 510, 508, 546, 505, 495, 530, 486, 520, 510, 528, 518, 513, 520, 532, 501, 509, 523, 491, 494, 539, 505, 532, 521, 514, 502, 518, 515, 511, 517, 526, 506, 500, 486, 513, 527, 486, 492, 471, 574, 512, 478, 532] payload_rows_per_dst=[5572, 5358, 5772, 5410, 5530, 5534, 5588, 5658] dup_rows_per_dst=[2626, 2354, 2858, 2398, 2684, 2618, 2658, 2918] combine_rows_per_src=[5572, 5358, 5772, 5410, 5530, 5534, 5588, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229592 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279236824 ingress_bytes=499208192 link_bytes=499208192 token_bytes=558244056 weight_movement_bytes=220200960 total_nvlink_bytes=778445016 weight_to_token_ratio=0.394453 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682051515e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584244464 prefetch_bytes=1101004800 reduce_bytes=990904320 weight_movement_bytes=2091909120 egress_bytes=2793268976 ingress_bytes=4882884608 link_bytes=4882884608 total_nvlink_bytes=7676153584 weight_to_token_ratio=0.374609 prefetch_share_of_ingress=0.225482 dedup_ratio=0.320959 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=9 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.492139430e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38782 dup_rows=21000 dup_rows_remote=18022 dedup_ratio=0.320435 router_tokens_per_expert=[512, 535, 538, 548, 519, 519, 503, 486, 501, 475, 552, 506, 505, 539, 519, 532, 481, 498, 491, 507, 521, 506, 481, 513, 525, 474, 527, 530, 520, 543, 547, 517, 499, 489, 533, 547, 508, 528, 515, 524, 494, 465, 497, 531, 483, 532, 521, 534, 488, 518, 496, 523, 532, 495, 514, 510, 498, 498, 500, 485, 531, 500, 513, 497] payload_rows_per_dst=[5754, 5584, 5466, 5656, 5566, 5554, 5388, 5568] dup_rows_per_dst=[2978, 2736, 2530, 2658, 2758, 2512, 2352, 2476] combine_rows_per_src=[5754, 5584, 5466, 5656, 5566, 5554, 5388, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44556 payload_rows_remote=39010 dup_rows=20980 dup_rows_remote=18476 dedup_ratio=0.320129 router_tokens_per_expert=[542, 489, 535, 515, 485, 490, 484, 473, 544, 505, 489, 497, 503, 480, 535, 517, 534, 508, 516, 491, 515, 536, 508, 527, 531, 487, 530, 499, 500, 547, 503, 533, 566, 475, 478, 539, 526, 497, 515, 499, 506, 502, 507, 503, 499, 500, 515, 493, 534, 530, 532, 465, 530, 524, 509, 504, 527, 542, 528, 512, 505, 526, 495, 537] payload_rows_per_dst=[5556, 5542, 5546, 5604, 5552, 5546, 5572, 5638] dup_rows_per_dst=[2470, 2598, 2724, 2656, 2638, 2504, 2684, 2706] combine_rows_per_src=[5556, 5542, 5546, 5604, 5552, 5546, 5572, 5638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=38962 dup_rows=21036 dup_rows_remote=18460 dedup_ratio=0.320984 router_tokens_per_expert=[518, 529, 509, 461, 507, 504, 486, 469, 515, 553, 491, 524, 498, 496, 490, 490, 505, 548, 499, 500, 496, 496, 528, 482, 558, 513, 503, 514, 546, 539, 523, 521, 551, 490, 526, 526, 501, 548, 530, 515, 489, 506, 559, 543, 519, 508, 483, 490, 509, 493, 523, 506, 514, 519, 529, 510, 517, 505, 510, 485, 501, 514, 548, 490] payload_rows_per_dst=[5518, 5538, 5508, 5624, 5692, 5562, 5554, 5504] dup_rows_per_dst=[2448, 2576, 2600, 2810, 2682, 2632, 2652, 2636] combine_rows_per_src=[5518, 5538, 5508, 5624, 5692, 5562, 5554, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278849536 dispatch_meta_bytes=229704 combine_payload_bytes=278849536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279079240 ingress_bytes=388950016 link_bytes=388950016 token_bytes=557928776 weight_movement_bytes=110100480 total_nvlink_bytes=668029256 weight_to_token_ratio=0.197338 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764480491e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=229368 combine_payload_bytes=279552000 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279781368 ingress_bytes=609853440 link_bytes=609853440 token_bytes=559333368 weight_movement_bytes=330301440 total_nvlink_bytes=889634808 weight_to_token_ratio=0.590527 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060635436e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229592 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279236824 ingress_bytes=499208192 link_bytes=499208192 token_bytes=558244056 weight_movement_bytes=220200960 total_nvlink_bytes=778445016 weight_to_token_ratio=0.394453 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682051515e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5592531704 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=2797413112 ingress_bytes=4336525312 link_bytes=4336525312 total_nvlink_bytes=7133938424 weight_to_token_ratio=0.275619 prefetch_share_of_ingress=0.228502 dedup_ratio=0.320264 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.541930753e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5590096456 prefetch_bytes=660602880 reduce_bytes=1431306240 weight_movement_bytes=2091909120 egress_bytes=2796196424 ingress_bytes=4885809152 link_bytes=4885809152 total_nvlink_bytes=7682005576 weight_to_token_ratio=0.374217 prefetch_share_of_ingress=0.135208 dedup_ratio=0.320932 prefetch_slots=12 unused_prefetch_slots=148 slot_utilization=0.075000 reduce_slots=13 remote_demand=6 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.497225693e-05 prefetch_bytes_per_gemm_flop=1.148897059e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279967744 dispatch_meta_bytes=229432 combine_payload_bytes=279967744 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280197176 ingress_bytes=610269184 link_bytes=610269184 token_bytes=560164920 weight_movement_bytes=330301440 total_nvlink_bytes=890466360 weight_to_token_ratio=0.589650 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061358483e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5578624728 prefetch_bytes=880803840 reduce_bytes=1321205760 weight_movement_bytes=2202009600 egress_bytes=2790459096 ingress_bytes=4990175232 link_bytes=4990175232 total_nvlink_bytes=7780634328 weight_to_token_ratio=0.394723 prefetch_share_of_ingress=0.176508 dedup_ratio=0.321713 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=12 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.678735471e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277860352 dispatch_meta_bytes=227840 combine_payload_bytes=277860352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278088192 ingress_bytes=387960832 link_bytes=387960832 token_bytes=555948544 weight_movement_bytes=110100480 total_nvlink_bytes=666049024 weight_to_token_ratio=0.198041 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747276954e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278061056 dispatch_meta_bytes=230024 combine_payload_bytes=278061056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278291080 ingress_bytes=388161536 link_bytes=388161536 token_bytes=556352136 weight_movement_bytes=110100480 total_nvlink_bytes=666452616 weight_to_token_ratio=0.197897 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750767527e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5584244464 prefetch_bytes=1101004800 reduce_bytes=990904320 weight_movement_bytes=2091909120 egress_bytes=2793268976 ingress_bytes=4882884608 link_bytes=4882884608 total_nvlink_bytes=7676153584 weight_to_token_ratio=0.374609 prefetch_share_of_ingress=0.225482 dedup_ratio=0.320959 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=9 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.492139430e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277989376 dispatch_meta_bytes=227216 combine_payload_bytes=277989376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278216592 ingress_bytes=388089856 link_bytes=388089856 token_bytes=556205968 weight_movement_bytes=110100480 total_nvlink_bytes=666306448 weight_to_token_ratio=0.197949 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.749520894e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5578624728 prefetch_bytes=880803840 reduce_bytes=1321205760 weight_movement_bytes=2202009600 egress_bytes=2790459096 ingress_bytes=4990175232 link_bytes=4990175232 total_nvlink_bytes=7780634328 weight_to_token_ratio=0.394723 prefetch_share_of_ingress=0.176508 dedup_ratio=0.321713 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=12 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.678735471e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582555360 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=2792425696 ingress_bytes=4551737344 link_bytes=4551737344 total_nvlink_bytes=7344163040 weight_to_token_ratio=0.315556 prefetch_share_of_ingress=0.217698 dedup_ratio=0.321255 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.916219873e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5577164200 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=2789729704 ingress_bytes=4659142656 link_bytes=4659142656 total_nvlink_bytes=7448872360 weight_to_token_ratio=0.335602 prefetch_share_of_ingress=0.212680 dedup_ratio=0.321945 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.103015376e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=229944 combine_payload_bytes=279623680 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279853624 ingress_bytes=609925120 link_bytes=609925120 token_bytes=559477304 weight_movement_bytes=330301440 total_nvlink_bytes=889778744 weight_to_token_ratio=0.590375 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060760099e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279279616 dispatch_meta_bytes=229688 combine_payload_bytes=279279616 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279509304 ingress_bytes=499480576 link_bytes=499480576 token_bytes=558788920 weight_movement_bytes=220200960 total_nvlink_bytes=778989880 weight_to_token_ratio=0.394068 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.686788721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5566603368 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=2784444520 ingress_bytes=4984168448 link_bytes=4984168448 total_nvlink_bytes=7768612968 weight_to_token_ratio=0.395575 prefetch_share_of_ingress=0.198810 dedup_ratio=0.321954 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.668288686e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5567490152 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=2784886888 ingress_bytes=4434110464 link_bytes=4434110464 total_nvlink_bytes=7218997352 weight_to_token_ratio=0.296634 prefetch_share_of_ingress=0.248303 dedup_ratio=0.321674 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.711647383e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none - [2026-08-26 16:58:55] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 673.7 | throughput per GPU (TFLOP/s/GPU): 183.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.491002E+00 | loss scale: 1.0 | grad norm: 3.544 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=38994 dup_rows=20938 dup_rows_remote=18244 dedup_ratio=0.319489 router_tokens_per_expert=[507, 507, 517, 538, 521, 465, 523, 515, 513, 520, 509, 484, 516, 517, 534, 527, 536, 541, 520, 507, 504, 528, 514, 514, 546, 511, 489, 527, 552, 475, 510, 488, 495, 517, 512, 511, 573, 501, 544, 496, 514, 480, 510, 512, 550, 512, 503, 537, 493, 542, 525, 489, 462, 508, 498, 493, 483, 542, 487, 494, 491, 512, 508, 499] payload_rows_per_dst=[5618, 5514, 5618, 5552, 5604, 5578, 5568, 5546] dup_rows_per_dst=[2568, 2726, 2710, 2644, 2694, 2658, 2452, 2486] combine_rows_per_src=[5618, 5514, 5618, 5552, 5604, 5578, 5568, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5592531704 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=2797413112 ingress_bytes=4336525312 link_bytes=4336525312 total_nvlink_bytes=7133938424 weight_to_token_ratio=0.275619 prefetch_share_of_ingress=0.228502 dedup_ratio=0.320264 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.541930753e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5590096456 prefetch_bytes=660602880 reduce_bytes=1431306240 weight_movement_bytes=2091909120 egress_bytes=2796196424 ingress_bytes=4885809152 link_bytes=4885809152 total_nvlink_bytes=7682005576 weight_to_token_ratio=0.374217 prefetch_share_of_ingress=0.135208 dedup_ratio=0.320932 prefetch_slots=12 unused_prefetch_slots=148 slot_utilization=0.075000 reduce_slots=13 remote_demand=6 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.497225693e-05 prefetch_bytes_per_gemm_flop=1.148897059e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44296 payload_rows_remote=38792 dup_rows=21240 dup_rows_remote=18714 dedup_ratio=0.324097 router_tokens_per_expert=[508, 532, 464, 515, 485, 509, 528, 518, 512, 528, 537, 567, 534, 523, 521, 487, 524, 515, 521, 501, 475, 488, 478, 513, 513, 511, 505, 516, 478, 550, 517, 534, 487, 484, 510, 515, 514, 517, 513, 484, 537, 514, 493, 503, 528, 478, 537, 553, 469, 539, 532, 496, 529, 517, 532, 522, 499, 534, 540, 514, 454, 473, 519, 525] payload_rows_per_dst=[5444, 5706, 5504, 5500, 5470, 5514, 5578, 5580] dup_rows_per_dst=[2674, 2712, 2526, 2748, 2578, 2772, 2694, 2536] combine_rows_per_src=[5444, 5706, 5504, 5500, 5470, 5514, 5578, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278061056 dispatch_meta_bytes=230024 combine_payload_bytes=278061056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278291080 ingress_bytes=388161536 link_bytes=388161536 token_bytes=556352136 weight_movement_bytes=110100480 total_nvlink_bytes=666452616 weight_to_token_ratio=0.197897 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750767527e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5582555360 prefetch_bytes=990904320 reduce_bytes=770703360 weight_movement_bytes=1761607680 egress_bytes=2792425696 ingress_bytes=4551737344 link_bytes=4551737344 total_nvlink_bytes=7344163040 weight_to_token_ratio=0.315556 prefetch_share_of_ingress=0.217698 dedup_ratio=0.321255 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=7 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.916219873e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none + [2026-08-26 18:11:00] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 653.1 | throughput per GPU (TFLOP/s/GPU): 189.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.489799E+00 | loss scale: 1.0 | grad norm: 3.487 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44692 payload_rows_remote=39138 dup_rows=20844 dup_rows_remote=18172 dedup_ratio=0.318054 router_tokens_per_expert=[499, 540, 527, 521, 523, 509, 487, 549, 503, 500, 519, 494, 500, 503, 513, 507, 524, 479, 589, 489, 478, 510, 518, 470, 526, 502, 515, 522, 530, 519, 487, 525, 509, 514, 531, 520, 454, 470, 529, 470, 499, 511, 519, 526, 505, 569, 491, 493, 531, 525, 523, 514, 534, 514, 537, 528, 543, 501, 512, 534, 499, 493, 478, 515] payload_rows_per_dst=[5646, 5546, 5530, 5662, 5482, 5554, 5698, 5574] dup_rows_per_dst=[2664, 2532, 2584, 2590, 2512, 2672, 2714, 2576] combine_rows_per_src=[5646, 5546, 5530, 5662, 5482, 5554, 5698, 5574] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44504 payload_rows_remote=39092 dup_rows=21032 dup_rows_remote=18476 dedup_ratio=0.320923 router_tokens_per_expert=[507, 479, 530, 513, 494, 503, 496, 515, 498, 497, 532, 550, 500, 501, 514, 540, 527, 493, 477, 541, 496, 501, 509, 475, 537, 486, 511, 552, 500, 493, 514, 502, 530, 526, 525, 505, 538, 559, 560, 516, 553, 512, 481, 489, 537, 528, 556, 517, 464, 495, 491, 547, 471, 497, 522, 497, 522, 494, 528, 486, 484, 533, 522, 500] payload_rows_per_dst=[5558, 5550, 5548, 5552, 5740, 5582, 5412, 5562] dup_rows_per_dst=[2516, 2714, 2490, 2638, 2778, 2764, 2556, 2576] combine_rows_per_src=[5558, 5550, 5548, 5552, 5740, 5582, 5412, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=39034 dup_rows=21014 dup_rows_remote=18492 dedup_ratio=0.320648 router_tokens_per_expert=[516, 502, 506, 559, 564, 532, 504, 529, 505, 553, 492, 467, 498, 520, 483, 531, 524, 534, 530, 540, 521, 488, 496, 507, 505, 496, 484, 500, 513, 503, 504, 500, 535, 502, 486, 492, 495, 506, 521, 498, 524, 507, 555, 491, 542, 516, 493, 500, 481, 489, 471, 516, 502, 525, 503, 535, 504, 490, 539, 530, 498, 527, 562, 527] payload_rows_per_dst=[5682, 5618, 5602, 5488, 5506, 5566, 5500, 5560] dup_rows_per_dst=[2742, 2480, 2678, 2522, 2564, 2690, 2544, 2794] combine_rows_per_src=[5682, 5618, 5602, 5488, 5506, 5566, 5500, 5560] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=39102 dup_rows=20954 dup_rows_remote=18404 dedup_ratio=0.319733 router_tokens_per_expert=[480, 516, 482, 529, 509, 545, 513, 534, 512, 444, 542, 505, 510, 489, 519, 494, 467, 498, 534, 512, 485, 531, 484, 518, 554, 533, 482, 516, 561, 519, 518, 541, 526, 521, 480, 529, 504, 509, 532, 517, 512, 554, 513, 510, 510, 516, 505, 509, 528, 495, 558, 488, 508, 516, 499, 503, 526, 516, 520, 505, 504, 504, 484, 491] payload_rows_per_dst=[5604, 5480, 5536, 5704, 5550, 5570, 5564, 5574] dup_rows_per_dst=[2612, 2550, 2522, 2744, 2686, 2688, 2626, 2526] combine_rows_per_src=[5604, 5480, 5536, 5704, 5550, 5570, 5564, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44692 payload_rows_remote=39138 dup_rows=20844 dup_rows_remote=18172 dedup_ratio=0.318054 router_tokens_per_expert=[499, 540, 527, 521, 523, 509, 487, 549, 503, 500, 519, 494, 500, 503, 513, 507, 524, 479, 589, 489, 478, 510, 518, 470, 526, 502, 515, 522, 530, 519, 487, 525, 509, 514, 531, 520, 454, 470, 529, 470, 499, 511, 519, 526, 505, 569, 491, 493, 531, 525, 523, 514, 534, 514, 537, 528, 543, 501, 512, 534, 499, 493, 478, 515] payload_rows_per_dst=[5646, 5546, 5530, 5662, 5482, 5554, 5698, 5574] dup_rows_per_dst=[2664, 2532, 2584, 2590, 2512, 2672, 2714, 2576] combine_rows_per_src=[5646, 5546, 5530, 5662, 5482, 5554, 5698, 5574] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38758 dup_rows=21160 dup_rows_remote=18318 dedup_ratio=0.322876 router_tokens_per_expert=[495, 524, 499, 528, 530, 496, 509, 489, 524, 510, 514, 499, 484, 534, 517, 486, 489, 514, 556, 497, 521, 513, 531, 511, 499, 533, 510, 499, 523, 521, 512, 521, 493, 501, 521, 505, 521, 529, 518, 518, 520, 530, 533, 504, 483, 504, 525, 493, 540, 529, 498, 542, 488, 473, 516, 513, 492, 497, 540, 522, 514, 502, 515, 501] payload_rows_per_dst=[5486, 5464, 5556, 5434, 5588, 5558, 5672, 5618] dup_rows_per_dst=[2398, 2672, 2708, 2490, 2642, 2626, 2782, 2842] combine_rows_per_src=[5486, 5464, 5556, 5434, 5588, 5558, 5672, 5618] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=38994 dup_rows=20938 dup_rows_remote=18244 dedup_ratio=0.319489 router_tokens_per_expert=[507, 507, 517, 538, 521, 465, 523, 515, 513, 520, 509, 484, 516, 517, 534, 527, 536, 541, 520, 507, 504, 528, 514, 514, 546, 511, 489, 527, 552, 475, 510, 488, 495, 517, 512, 511, 573, 501, 544, 496, 514, 480, 510, 512, 550, 512, 503, 537, 493, 542, 525, 489, 462, 508, 498, 493, 483, 542, 487, 494, 491, 512, 508, 499] payload_rows_per_dst=[5618, 5514, 5618, 5552, 5604, 5578, 5568, 5546] dup_rows_per_dst=[2568, 2726, 2710, 2644, 2694, 2658, 2452, 2486] combine_rows_per_src=[5618, 5514, 5618, 5552, 5604, 5578, 5568, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44556 payload_rows_remote=38956 dup_rows=20980 dup_rows_remote=18520 dedup_ratio=0.320129 router_tokens_per_expert=[546, 534, 495, 506, 525, 540, 532, 508, 490, 522, 551, 533, 489, 548, 547, 481, 518, 483, 503, 474, 510, 526, 486, 530, 519, 535, 497, 480, 560, 532, 524, 538, 499, 507, 504, 513, 508, 489, 514, 511, 505, 493, 478, 452, 533, 500, 520, 518, 532, 514, 463, 549, 544, 529, 505, 511, 520, 553, 520, 487, 507, 505, 475, 448] payload_rows_per_dst=[5592, 5624, 5600, 5678, 5518, 5476, 5552, 5516] dup_rows_per_dst=[2780, 2698, 2460, 2692, 2572, 2522, 2742, 2514] combine_rows_per_src=[5592, 5624, 5600, 5678, 5518, 5476, 5552, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44522 payload_rows_remote=39034 dup_rows=21014 dup_rows_remote=18492 dedup_ratio=0.320648 router_tokens_per_expert=[516, 502, 506, 559, 564, 532, 504, 529, 505, 553, 492, 467, 498, 520, 483, 531, 524, 534, 530, 540, 521, 488, 496, 507, 505, 496, 484, 500, 513, 503, 504, 500, 535, 502, 486, 492, 495, 506, 521, 498, 524, 507, 555, 491, 542, 516, 493, 500, 481, 489, 471, 516, 502, 525, 503, 535, 504, 490, 539, 530, 498, 527, 562, 527] payload_rows_per_dst=[5682, 5618, 5602, 5488, 5506, 5566, 5500, 5560] dup_rows_per_dst=[2742, 2480, 2678, 2522, 2564, 2690, 2544, 2794] combine_rows_per_src=[5682, 5618, 5602, 5488, 5506, 5566, 5500, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44556 payload_rows_remote=38956 dup_rows=20980 dup_rows_remote=18520 dedup_ratio=0.320129 router_tokens_per_expert=[546, 534, 495, 506, 525, 540, 532, 508, 490, 522, 551, 533, 489, 548, 547, 481, 518, 483, 503, 474, 510, 526, 486, 530, 519, 535, 497, 480, 560, 532, 524, 538, 499, 507, 504, 513, 508, 489, 514, 511, 505, 493, 478, 452, 533, 500, 520, 518, 532, 514, 463, 549, 544, 529, 505, 511, 520, 553, 520, 487, 507, 505, 475, 448] payload_rows_per_dst=[5592, 5624, 5600, 5678, 5518, 5476, 5552, 5516] dup_rows_per_dst=[2780, 2698, 2460, 2692, 2572, 2522, 2742, 2514] combine_rows_per_src=[5592, 5624, 5600, 5678, 5518, 5476, 5552, 5516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=228952 combine_payload_bytes=279508992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279737944 ingress_bytes=389609472 link_bytes=389609472 token_bytes=559246936 weight_movement_bytes=110100480 total_nvlink_bytes=669347416 weight_to_token_ratio=0.196873 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775949516e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280211456 dispatch_meta_bytes=230272 combine_payload_bytes=280211456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280441728 ingress_bytes=390311936 link_bytes=390311936 token_bytes=560653184 weight_movement_bytes=110100480 total_nvlink_bytes=670753664 weight_to_token_ratio=0.196379 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788166520e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=230024 combine_payload_bytes=280283136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280513160 ingress_bytes=390383616 link_bytes=390383616 token_bytes=560796296 weight_movement_bytes=110100480 total_nvlink_bytes=670896776 weight_to_token_ratio=0.196329 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789413153e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=230104 combine_payload_bytes=279795712 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280025816 ingress_bytes=610097152 link_bytes=610097152 token_bytes=559821528 weight_movement_bytes=330301440 total_nvlink_bytes=890122968 weight_to_token_ratio=0.590012 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061059291e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38904 dup_rows=21114 dup_rows_remote=18562 dedup_ratio=0.322174 router_tokens_per_expert=[485, 538, 502, 511, 514, 461, 506, 502, 533, 560, 504, 520, 506, 496, 506, 510, 521, 508, 446, 489, 546, 499, 501, 532, 501, 517, 495, 515, 475, 541, 509, 536, 532, 504, 498, 486, 518, 505, 533, 474, 518, 557, 560, 515, 524, 496, 533, 517, 522, 458, 493, 513, 519, 531, 507, 534, 500, 517, 539, 499, 545, 531, 519, 486] payload_rows_per_dst=[5518, 5688, 5568, 5540, 5510, 5468, 5556, 5574] dup_rows_per_dst=[2552, 2680, 2826, 2606, 2590, 2564, 2598, 2698] combine_rows_per_src=[5518, 5688, 5568, 5540, 5510, 5468, 5556, 5574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280541184 dispatch_meta_bytes=229240 combine_payload_bytes=280541184 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280770424 ingress_bytes=500742144 link_bytes=500742144 token_bytes=561311608 weight_movement_bytes=220200960 total_nvlink_bytes=781512568 weight_to_token_ratio=0.392297 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.708729463e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280541184 dispatch_meta_bytes=229240 combine_payload_bytes=280541184 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=280770424 ingress_bytes=500742144 link_bytes=500742144 token_bytes=561311608 weight_movement_bytes=220200960 total_nvlink_bytes=781512568 weight_to_token_ratio=0.392297 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.708729463e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280211456 dispatch_meta_bytes=230272 combine_payload_bytes=280211456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280441728 ingress_bytes=390311936 link_bytes=390311936 token_bytes=560653184 weight_movement_bytes=110100480 total_nvlink_bytes=670753664 weight_to_token_ratio=0.196379 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.788166520e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280283136 dispatch_meta_bytes=230024 combine_payload_bytes=280283136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280513160 ingress_bytes=390383616 link_bytes=390383616 token_bytes=560796296 weight_movement_bytes=110100480 total_nvlink_bytes=670896776 weight_to_token_ratio=0.196329 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.789413153e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277817344 dispatch_meta_bytes=228304 combine_payload_bytes=277817344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278045648 ingress_bytes=387917824 link_bytes=387917824 token_bytes=555862992 weight_movement_bytes=110100480 total_nvlink_bytes=665963472 weight_to_token_ratio=0.198071 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746528975e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279508992 dispatch_meta_bytes=228952 combine_payload_bytes=279508992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279737944 ingress_bytes=389609472 link_bytes=389609472 token_bytes=559246936 weight_movement_bytes=110100480 total_nvlink_bytes=669347416 weight_to_token_ratio=0.196873 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775949516e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279236608 dispatch_meta_bytes=229904 combine_payload_bytes=279236608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279466512 ingress_bytes=389337088 link_bytes=389337088 token_bytes=558703120 weight_movement_bytes=110100480 total_nvlink_bytes=668803600 weight_to_token_ratio=0.197064 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.771212310e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=230104 combine_payload_bytes=279795712 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280025816 ingress_bytes=610097152 link_bytes=610097152 token_bytes=559821528 weight_movement_bytes=330301440 total_nvlink_bytes=890122968 weight_to_token_ratio=0.590012 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061059291e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278863872 dispatch_meta_bytes=229864 combine_payload_bytes=278863872 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279093736 ingress_bytes=499064832 link_bytes=499064832 token_bytes=557957608 weight_movement_bytes=220200960 total_nvlink_bytes=778158568 weight_to_token_ratio=0.394655 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679558249e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38828 dup_rows=21114 dup_rows_remote=18362 dedup_ratio=0.322174 router_tokens_per_expert=[514, 513, 497, 521, 526, 552, 523, 498, 522, 483, 505, 521, 552, 500, 522, 471, 507, 512, 486, 500, 523, 558, 518, 494, 534, 509, 481, 502, 495, 521, 499, 531, 537, 525, 511, 501, 538, 520, 527, 514, 497, 476, 473, 523, 505, 542, 518, 507, 510, 506, 515, 513, 494, 516, 508, 528, 518, 515, 503, 502, 502, 503, 513, 518] payload_rows_per_dst=[5580, 5478, 5572, 5560, 5594, 5548, 5544, 5546] dup_rows_per_dst=[2708, 2674, 2624, 2584, 2752, 2534, 2636, 2602] combine_rows_per_src=[5580, 5478, 5572, 5560, 5594, 5548, 5544, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44700 payload_rows_remote=39160 dup_rows=20836 dup_rows_remote=18368 dedup_ratio=0.317932 router_tokens_per_expert=[525, 532, 509, 478, 542, 530, 515, 525, 538, 458, 520, 540, 494, 539, 498, 546, 480, 496, 491, 509, 473, 502, 502, 510, 511, 515, 499, 563, 536, 508, 520, 536, 484, 498, 477, 520, 507, 505, 498, 492, 537, 521, 522, 507, 555, 490, 502, 522, 531, 496, 491, 519, 504, 478, 478, 507, 513, 502, 516, 546, 502, 539, 552, 517] payload_rows_per_dst=[5558, 5630, 5438, 5714, 5492, 5626, 5540, 5702] dup_rows_per_dst=[2754, 2636, 2488, 2662, 2470, 2686, 2468, 2672] combine_rows_per_src=[5558, 5630, 5438, 5714, 5492, 5626, 5540, 5702] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44628 payload_rows_remote=39148 dup_rows=20908 dup_rows_remote=18282 dedup_ratio=0.319031 router_tokens_per_expert=[512, 537, 548, 507, 519, 539, 494, 499, 490, 529, 512, 527, 512, 487, 492, 504, 509, 559, 486, 531, 510, 508, 509, 524, 514, 511, 531, 503, 554, 504, 489, 502, 497, 502, 547, 524, 530, 536, 511, 528, 479, 476, 522, 488, 510, 496, 548, 497, 484, 524, 528, 487, 509, 528, 500, 502, 544, 500, 506, 493, 490, 521, 483, 526] payload_rows_per_dst=[5662, 5480, 5620, 5572, 5600, 5558, 5544, 5592] dup_rows_per_dst=[2648, 2626, 2652, 2644, 2750, 2474, 2580, 2534] combine_rows_per_src=[5662, 5480, 5620, 5572, 5600, 5558, 5544, 5592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44404 payload_rows_remote=38840 dup_rows=21132 dup_rows_remote=18438 dedup_ratio=0.322449 router_tokens_per_expert=[506, 523, 533, 499, 533, 530, 475, 543, 496, 553, 515, 476, 517, 505, 568, 537, 516, 524, 510, 498, 521, 513, 510, 540, 514, 527, 525, 508, 552, 514, 481, 508, 533, 475, 508, 506, 509, 513, 506, 525, 492, 533, 521, 524, 509, 546, 532, 493, 483, 515, 502, 463, 503, 527, 496, 497, 485, 474, 489, 488, 549, 505, 500, 497] payload_rows_per_dst=[5592, 5596, 5606, 5564, 5582, 5558, 5458, 5448] dup_rows_per_dst=[2692, 2738, 2658, 2694, 2568, 2742, 2514, 2526] combine_rows_per_src=[5592, 5596, 5606, 5564, 5582, 5558, 5458, 5448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44700 payload_rows_remote=39160 dup_rows=20836 dup_rows_remote=18368 dedup_ratio=0.317932 router_tokens_per_expert=[525, 532, 509, 478, 542, 530, 515, 525, 538, 458, 520, 540, 494, 539, 498, 546, 480, 496, 491, 509, 473, 502, 502, 510, 511, 515, 499, 563, 536, 508, 520, 536, 484, 498, 477, 520, 507, 505, 498, 492, 537, 521, 522, 507, 555, 490, 502, 522, 531, 496, 491, 519, 504, 478, 478, 507, 513, 502, 516, 546, 502, 539, 552, 517] payload_rows_per_dst=[5558, 5630, 5438, 5714, 5492, 5626, 5540, 5702] dup_rows_per_dst=[2754, 2636, 2488, 2662, 2470, 2686, 2468, 2672] combine_rows_per_src=[5558, 5630, 5438, 5714, 5492, 5626, 5540, 5702] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44792 payload_rows_remote=39210 dup_rows=20744 dup_rows_remote=18060 dedup_ratio=0.316528 router_tokens_per_expert=[484, 509, 473, 525, 507, 521, 525, 513, 502, 533, 538, 522, 522, 464, 521, 459, 498, 517, 498, 534, 514, 498, 521, 486, 527, 520, 506, 489, 517, 505, 515, 499, 510, 493, 493, 513, 554, 525, 533, 510, 535, 531, 522, 519, 522, 506, 495, 503, 504, 515, 516, 508, 519, 515, 523, 519, 521, 543, 477, 523, 480, 520, 531, 528] payload_rows_per_dst=[5548, 5524, 5584, 5630, 5638, 5582, 5628, 5658] dup_rows_per_dst=[2566, 2598, 2548, 2526, 2624, 2684, 2610, 2588] combine_rows_per_src=[5548, 5524, 5584, 5630, 5638, 5582, 5628, 5658] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44316 payload_rows_remote=38800 dup_rows=21220 dup_rows_remote=18508 dedup_ratio=0.323792 router_tokens_per_expert=[507, 508, 498, 486, 499, 531, 514, 505, 500, 533, 566, 477, 489, 499, 528, 519, 546, 524, 501, 511, 535, 522, 525, 480, 522, 516, 475, 520, 513, 534, 523, 523, 465, 533, 502, 549, 547, 513, 487, 498, 496, 493, 539, 492, 552, 504, 501, 524, 488, 487, 527, 501, 527, 500, 543, 503, 543, 488, 486, 487, 520, 458, 545, 541] payload_rows_per_dst=[5248, 5554, 5566, 5554, 5582, 5558, 5738, 5516] dup_rows_per_dst=[2182, 2640, 2630, 2698, 2634, 2644, 3080, 2712] combine_rows_per_src=[5248, 5554, 5566, 5554, 5582, 5558, 5738, 5516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278319104 dispatch_meta_bytes=228760 combine_payload_bytes=278319104 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278547864 ingress_bytes=498520064 link_bytes=498520064 token_bytes=556866968 weight_movement_bytes=220200960 total_nvlink_bytes=777067928 weight_to_token_ratio=0.395428 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670083838e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280698880 dispatch_meta_bytes=230112 combine_payload_bytes=280698880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280928992 ingress_bytes=390799360 link_bytes=390799360 token_bytes=561627872 weight_movement_bytes=110100480 total_nvlink_bytes=671728352 weight_to_token_ratio=0.196038 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.796643625e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280612864 dispatch_meta_bytes=229720 combine_payload_bytes=280612864 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280842584 ingress_bytes=500813824 link_bytes=500813824 token_bytes=561455448 weight_movement_bytes=220200960 total_nvlink_bytes=781656408 weight_to_token_ratio=0.392197 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.709976097e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278405120 dispatch_meta_bytes=229112 combine_payload_bytes=278405120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278634232 ingress_bytes=498606080 link_bytes=498606080 token_bytes=557039352 weight_movement_bytes=220200960 total_nvlink_bytes=777240312 weight_to_token_ratio=0.395306 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671579797e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44422 payload_rows_remote=38828 dup_rows=21114 dup_rows_remote=18362 dedup_ratio=0.322174 router_tokens_per_expert=[514, 513, 497, 521, 526, 552, 523, 498, 522, 483, 505, 521, 552, 500, 522, 471, 507, 512, 486, 500, 523, 558, 518, 494, 534, 509, 481, 502, 495, 521, 499, 531, 537, 525, 511, 501, 538, 520, 527, 514, 497, 476, 473, 523, 505, 542, 518, 507, 510, 506, 515, 513, 494, 516, 508, 528, 518, 515, 503, 502, 502, 503, 513, 518] payload_rows_per_dst=[5580, 5478, 5572, 5560, 5594, 5548, 5544, 5546] dup_rows_per_dst=[2708, 2674, 2624, 2584, 2752, 2534, 2636, 2602] combine_rows_per_src=[5580, 5478, 5572, 5560, 5594, 5548, 5544, 5546] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=38878 dup_rows=21110 dup_rows_remote=18434 dedup_ratio=0.322113 router_tokens_per_expert=[508, 543, 532, 498, 555, 521, 532, 508, 503, 479, 517, 511, 490, 508, 491, 520, 541, 524, 514, 509, 509, 493, 510, 512, 510, 513, 526, 524, 507, 537, 517, 485, 526, 542, 520, 504, 522, 483, 482, 500, 497, 491, 528, 528, 454, 512, 514, 509, 521, 473, 510, 495, 517, 528, 504, 496, 501, 569, 532, 524, 507, 472, 539, 521] payload_rows_per_dst=[5616, 5490, 5548, 5554, 5490, 5566, 5542, 5620] dup_rows_per_dst=[2778, 2548, 2676, 2684, 2668, 2500, 2546, 2710] combine_rows_per_src=[5616, 5490, 5548, 5554, 5490, 5566, 5542, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44404 payload_rows_remote=38840 dup_rows=21132 dup_rows_remote=18438 dedup_ratio=0.322449 router_tokens_per_expert=[506, 523, 533, 499, 533, 530, 475, 543, 496, 553, 515, 476, 517, 505, 568, 537, 516, 524, 510, 498, 521, 513, 510, 540, 514, 527, 525, 508, 552, 514, 481, 508, 533, 475, 508, 506, 509, 513, 506, 525, 492, 533, 521, 524, 509, 546, 532, 493, 483, 515, 502, 463, 503, 527, 496, 497, 485, 474, 489, 488, 549, 505, 500, 497] payload_rows_per_dst=[5592, 5596, 5606, 5564, 5582, 5558, 5458, 5448] dup_rows_per_dst=[2692, 2738, 2658, 2694, 2568, 2742, 2514, 2526] combine_rows_per_src=[5592, 5596, 5606, 5564, 5582, 5558, 5458, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280612864 dispatch_meta_bytes=229720 combine_payload_bytes=280612864 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280842584 ingress_bytes=500813824 link_bytes=500813824 token_bytes=561455448 weight_movement_bytes=220200960 total_nvlink_bytes=781656408 weight_to_token_ratio=0.392197 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.709976097e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=280698880 dispatch_meta_bytes=230112 combine_payload_bytes=280698880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280928992 ingress_bytes=390799360 link_bytes=390799360 token_bytes=561627872 weight_movement_bytes=110100480 total_nvlink_bytes=671728352 weight_to_token_ratio=0.196038 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.796643625e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=281057280 dispatch_meta_bytes=229080 combine_payload_bytes=281057280 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=281286360 ingress_bytes=501258240 link_bytes=501258240 token_bytes=562343640 weight_movement_bytes=220200960 total_nvlink_bytes=782544600 weight_to_token_ratio=0.391577 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.717705222e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278118400 dispatch_meta_bytes=229232 combine_payload_bytes=278118400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278347632 ingress_bytes=388218880 link_bytes=388218880 token_bytes=556466032 weight_movement_bytes=110100480 total_nvlink_bytes=666566512 weight_to_token_ratio=0.197857 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751764834e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278118400 dispatch_meta_bytes=229232 combine_payload_bytes=278118400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278347632 ingress_bytes=388218880 link_bytes=388218880 token_bytes=556466032 weight_movement_bytes=110100480 total_nvlink_bytes=666566512 weight_to_token_ratio=0.197857 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.751764834e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44548 payload_rows_remote=38892 dup_rows=20988 dup_rows_remote=18240 dedup_ratio=0.320251 router_tokens_per_expert=[523, 483, 526, 485, 487, 538, 498, 514, 517, 487, 522, 541, 544, 514, 540, 493, 533, 514, 496, 540, 494, 550, 499, 533, 500, 487, 526, 527, 525, 521, 472, 513, 497, 494, 524, 540, 495, 506, 511, 504, 489, 497, 519, 521, 519, 504, 535, 562, 499, 499, 527, 466, 534, 527, 506, 496, 523, 529, 498, 508, 527, 522, 461, 487] payload_rows_per_dst=[5656, 5652, 5628, 5528, 5564, 5500, 5540, 5480] dup_rows_per_dst=[2748, 2672, 2698, 2368, 2528, 2776, 2568, 2630] combine_rows_per_src=[5656, 5652, 5628, 5528, 5564, 5500, 5540, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278319104 dispatch_meta_bytes=228760 combine_payload_bytes=278319104 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278547864 ingress_bytes=498520064 link_bytes=498520064 token_bytes=556866968 weight_movement_bytes=220200960 total_nvlink_bytes=777067928 weight_to_token_ratio=0.395428 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670083838e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278677504 dispatch_meta_bytes=229248 combine_payload_bytes=278677504 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278906752 ingress_bytes=498878464 link_bytes=498878464 token_bytes=557584256 weight_movement_bytes=220200960 total_nvlink_bytes=777785216 weight_to_token_ratio=0.394920 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676317003e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278405120 dispatch_meta_bytes=229112 combine_payload_bytes=278405120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278634232 ingress_bytes=498606080 link_bytes=498606080 token_bytes=557039352 weight_movement_bytes=220200960 total_nvlink_bytes=777240312 weight_to_token_ratio=0.395306 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671579797e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38954 dup_rows=21058 dup_rows_remote=18470 dedup_ratio=0.321320 router_tokens_per_expert=[480, 518, 551, 500, 518, 520, 538, 496, 518, 499, 492, 506, 484, 505, 500, 521, 499, 507, 513, 544, 528, 469, 491, 453, 524, 505, 487, 565, 563, 529, 495, 535, 492, 487, 573, 497, 506, 480, 517, 504, 498, 502, 562, 538, 488, 515, 511, 488, 521, 532, 509, 516, 512, 505, 520, 536, 528, 502, 497, 531, 544, 523, 475, 506] payload_rows_per_dst=[5622, 5494, 5454, 5650, 5524, 5592, 5606, 5536] dup_rows_per_dst=[2620, 2556, 2554, 2756, 2588, 2612, 2696, 2676] combine_rows_per_src=[5622, 5494, 5454, 5650, 5524, 5592, 5606, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278777856 dispatch_meta_bytes=228528 combine_payload_bytes=278777856 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279006384 ingress_bytes=498978816 link_bytes=498978816 token_bytes=557784240 weight_movement_bytes=220200960 total_nvlink_bytes=777985200 weight_to_token_ratio=0.394778 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.678062289e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=39022 dup_rows=21088 dup_rows_remote=18442 dedup_ratio=0.321777 router_tokens_per_expert=[513, 493, 516, 523, 543, 501, 515, 536, 493, 520, 544, 527, 498, 489, 533, 523, 541, 532, 546, 498, 534, 508, 513, 475, 515, 482, 511, 500, 573, 510, 500, 512, 539, 501, 530, 519, 495, 554, 523, 500, 488, 545, 493, 532, 505, 480, 517, 473, 500, 518, 515, 515, 522, 502, 466, 498, 468, 533, 547, 486, 472, 467, 512, 536] payload_rows_per_dst=[5584, 5672, 5550, 5590, 5618, 5494, 5426, 5514] dup_rows_per_dst=[2696, 2582, 2744, 2616, 2704, 2572, 2646, 2528] combine_rows_per_src=[5584, 5672, 5550, 5590, 5618, 5494, 5426, 5514] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44622 payload_rows_remote=38948 dup_rows=20914 dup_rows_remote=18190 dedup_ratio=0.319122 router_tokens_per_expert=[493, 468, 513, 540, 527, 523, 478, 537, 503, 551, 556, 509, 490, 580, 472, 538, 490, 530, 527, 518, 505, 523, 503, 526, 492, 499, 529, 533, 500, 537, 483, 508, 494, 457, 526, 461, 499, 535, 504, 506, 524, 489, 502, 530, 499, 494, 507, 474, 520, 525, 503, 526, 491, 519, 511, 515, 505, 535, 531, 534, 526, 505, 525, 515] payload_rows_per_dst=[5546, 5674, 5570, 5580, 5552, 5528, 5590, 5582] dup_rows_per_dst=[2612, 2724, 2674, 2582, 2412, 2510, 2630, 2770] combine_rows_per_src=[5546, 5674, 5570, 5580, 5552, 5528, 5590, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44580 payload_rows_remote=38996 dup_rows=20956 dup_rows_remote=18218 dedup_ratio=0.319763 router_tokens_per_expert=[506, 503, 518, 545, 507, 504, 514, 495, 526, 514, 501, 492, 526, 504, 467, 495, 537, 510, 518, 491, 493, 510, 482, 509, 517, 509, 522, 513, 529, 521, 514, 536, 529, 488, 526, 517, 505, 542, 536, 516, 566, 514, 523, 457, 514, 495, 501, 524, 506, 515, 526, 520, 512, 510, 485, 514, 503, 519, 578, 477, 488, 524, 492, 518] payload_rows_per_dst=[5566, 5602, 5466, 5584, 5660, 5572, 5566, 5564] dup_rows_per_dst=[2618, 2448, 2634, 2738, 2658, 2616, 2610, 2634] combine_rows_per_src=[5566, 5602, 5466, 5584, 5660, 5572, 5566, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229696 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279451968 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558674240 weight_movement_bytes=110100480 total_nvlink_bytes=668774720 weight_to_token_ratio=0.197075 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=39022 dup_rows=21088 dup_rows_remote=18442 dedup_ratio=0.321777 router_tokens_per_expert=[513, 493, 516, 523, 543, 501, 515, 536, 493, 520, 544, 527, 498, 489, 533, 523, 541, 532, 546, 498, 534, 508, 513, 475, 515, 482, 511, 500, 573, 510, 500, 512, 539, 501, 530, 519, 495, 554, 523, 500, 488, 545, 493, 532, 505, 480, 517, 473, 500, 518, 515, 515, 522, 502, 466, 498, 468, 533, 547, 486, 472, 467, 512, 536] payload_rows_per_dst=[5584, 5672, 5550, 5590, 5618, 5494, 5426, 5514] dup_rows_per_dst=[2696, 2582, 2744, 2616, 2704, 2572, 2646, 2528] combine_rows_per_src=[5584, 5672, 5550, 5590, 5618, 5494, 5426, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=39012 dup_rows=21036 dup_rows_remote=18380 dedup_ratio=0.320984 router_tokens_per_expert=[523, 535, 511, 527, 535, 521, 540, 513, 526, 465, 512, 515, 532, 478, 515, 535, 507, 486, 517, 510, 497, 519, 573, 519, 509, 477, 555, 486, 535, 552, 521, 504, 496, 480, 485, 481, 493, 488, 497, 482, 490, 510, 497, 502, 515, 571, 482, 505, 523, 479, 558, 536, 537, 536, 491, 506, 509, 512, 502, 532, 495, 503, 516, 509] payload_rows_per_dst=[5610, 5538, 5578, 5646, 5416, 5488, 5658, 5566] dup_rows_per_dst=[2800, 2618, 2678, 2632, 2388, 2656, 2674, 2590] combine_rows_per_src=[5610, 5538, 5578, 5646, 5416, 5488, 5658, 5566] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=38906 dup_rows=21086 dup_rows_remote=18542 dedup_ratio=0.321747 router_tokens_per_expert=[562, 478, 505, 512, 470, 497, 501, 479, 475, 546, 492, 533, 503, 509, 486, 483, 537, 517, 546, 541, 514, 521, 477, 510, 549, 531, 524, 519, 521, 514, 509, 537, 489, 554, 539, 534, 490, 508, 520, 523, 519, 530, 522, 493, 516, 553, 519, 532, 489, 504, 515, 459, 531, 470, 508, 513, 469, 505, 534, 489, 505, 502, 512, 524] payload_rows_per_dst=[5478, 5508, 5600, 5384, 5614, 5782, 5540, 5544] dup_rows_per_dst=[2472, 2604, 2726, 2452, 2848, 2838, 2602, 2544] combine_rows_per_src=[5478, 5508, 5600, 5384, 5614, 5782, 5540, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=39012 dup_rows=21036 dup_rows_remote=18380 dedup_ratio=0.320984 router_tokens_per_expert=[523, 535, 511, 527, 535, 521, 540, 513, 526, 465, 512, 515, 532, 478, 515, 535, 507, 486, 517, 510, 497, 519, 573, 519, 509, 477, 555, 486, 535, 552, 521, 504, 496, 480, 485, 481, 493, 488, 497, 482, 490, 510, 497, 502, 515, 571, 482, 505, 523, 479, 558, 536, 537, 536, 491, 506, 509, 512, 502, 532, 495, 503, 516, 509] payload_rows_per_dst=[5610, 5538, 5578, 5646, 5416, 5488, 5658, 5566] dup_rows_per_dst=[2800, 2618, 2678, 2632, 2388, 2656, 2674, 2590] combine_rows_per_src=[5610, 5538, 5578, 5646, 5416, 5488, 5658, 5566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279709696 dispatch_meta_bytes=229856 combine_payload_bytes=279709696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279939552 ingress_bytes=389810176 link_bytes=389810176 token_bytes=559649248 weight_movement_bytes=110100480 total_nvlink_bytes=669749728 weight_to_token_ratio=0.196731 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779440088e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44396 payload_rows_remote=38906 dup_rows=21140 dup_rows_remote=18514 dedup_ratio=0.322571 router_tokens_per_expert=[506, 519, 493, 505, 504, 543, 513, 484, 516, 494, 504, 509, 499, 553, 527, 524, 510, 496, 510, 507, 511, 475, 520, 529, 500, 532, 508, 509, 537, 534, 469, 512, 537, 487, 507, 506, 539, 536, 535, 477, 538, 522, 504, 478, 498, 541, 481, 524, 523, 507, 462, 535, 483, 527, 503, 500, 518, 504, 521, 532, 520, 527, 519, 525] payload_rows_per_dst=[5554, 5524, 5490, 5614, 5520, 5596, 5508, 5590] dup_rows_per_dst=[2580, 2728, 2626, 2588, 2728, 2576, 2572, 2742] combine_rows_per_src=[5554, 5524, 5490, 5614, 5520, 5596, 5508, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44580 payload_rows_remote=38996 dup_rows=20956 dup_rows_remote=18218 dedup_ratio=0.319763 router_tokens_per_expert=[506, 503, 518, 545, 507, 504, 514, 495, 526, 514, 501, 492, 526, 504, 467, 495, 537, 510, 518, 491, 493, 510, 482, 509, 517, 509, 522, 513, 529, 521, 514, 536, 529, 488, 526, 517, 505, 542, 536, 516, 566, 514, 523, 457, 514, 495, 501, 524, 506, 515, 526, 520, 512, 510, 485, 514, 503, 519, 578, 477, 488, 524, 492, 518] payload_rows_per_dst=[5566, 5602, 5466, 5584, 5660, 5572, 5566, 5564] dup_rows_per_dst=[2618, 2448, 2634, 2738, 2658, 2616, 2610, 2634] combine_rows_per_src=[5566, 5602, 5466, 5584, 5660, 5572, 5566, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38954 dup_rows=21058 dup_rows_remote=18470 dedup_ratio=0.321320 router_tokens_per_expert=[480, 518, 551, 500, 518, 520, 538, 496, 518, 499, 492, 506, 484, 505, 500, 521, 499, 507, 513, 544, 528, 469, 491, 453, 524, 505, 487, 565, 563, 529, 495, 535, 492, 487, 573, 497, 506, 480, 517, 504, 498, 502, 562, 538, 488, 515, 511, 488, 521, 532, 509, 516, 512, 505, 520, 536, 528, 502, 497, 531, 544, 523, 475, 506] payload_rows_per_dst=[5622, 5494, 5454, 5650, 5524, 5592, 5606, 5536] dup_rows_per_dst=[2620, 2556, 2554, 2756, 2588, 2612, 2696, 2676] combine_rows_per_src=[5622, 5494, 5454, 5650, 5524, 5592, 5606, 5536] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279179264 dispatch_meta_bytes=228552 combine_payload_bytes=279179264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279407816 ingress_bytes=389279744 link_bytes=389279744 token_bytes=558587080 weight_movement_bytes=110100480 total_nvlink_bytes=668687560 weight_to_token_ratio=0.197105 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770215003e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279638016 dispatch_meta_bytes=229568 combine_payload_bytes=279638016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279867584 ingress_bytes=389738496 link_bytes=389738496 token_bytes=559505600 weight_movement_bytes=110100480 total_nvlink_bytes=669606080 weight_to_token_ratio=0.196782 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778193455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279709696 dispatch_meta_bytes=229856 combine_payload_bytes=279709696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279939552 ingress_bytes=389810176 link_bytes=389810176 token_bytes=559649248 weight_movement_bytes=110100480 total_nvlink_bytes=669749728 weight_to_token_ratio=0.196731 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.779440088e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44396 payload_rows_remote=38906 dup_rows=21140 dup_rows_remote=18514 dedup_ratio=0.322571 router_tokens_per_expert=[506, 519, 493, 505, 504, 543, 513, 484, 516, 494, 504, 509, 499, 553, 527, 524, 510, 496, 510, 507, 511, 475, 520, 529, 500, 532, 508, 509, 537, 534, 469, 512, 537, 487, 507, 506, 539, 536, 535, 477, 538, 522, 504, 478, 498, 541, 481, 524, 523, 507, 462, 535, 483, 527, 503, 500, 518, 504, 521, 532, 520, 527, 519, 525] payload_rows_per_dst=[5554, 5524, 5490, 5614, 5520, 5596, 5508, 5590] dup_rows_per_dst=[2580, 2728, 2626, 2588, 2728, 2576, 2572, 2742] combine_rows_per_src=[5554, 5524, 5490, 5614, 5520, 5596, 5508, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=228856 combine_payload_bytes=279523328 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=279752184 ingress_bytes=830025728 link_bytes=830025728 token_bytes=559275512 weight_movement_bytes=550502400 total_nvlink_bytes=1109777912 weight_to_token_ratio=0.984313 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.443551257e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=229792 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279108000 ingress_bytes=499079168 link_bytes=499079168 token_bytes=557986208 weight_movement_bytes=220200960 total_nvlink_bytes=778187168 weight_to_token_ratio=0.394635 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679807576e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44286 payload_rows_remote=38762 dup_rows=21250 dup_rows_remote=18596 dedup_ratio=0.324249 router_tokens_per_expert=[495, 486, 517, 484, 490, 523, 570, 522, 545, 544, 497, 519, 533, 487, 514, 493, 491, 542, 505, 508, 504, 486, 526, 507, 514, 501, 563, 488, 506, 516, 502, 474, 522, 542, 541, 544, 516, 505, 488, 495, 521, 520, 494, 499, 501, 487, 504, 526, 508, 537, 489, 497, 500, 570, 500, 505, 502, 485, 530, 492, 530, 541, 520, 505] payload_rows_per_dst=[5524, 5584, 5510, 5494, 5538, 5456, 5584, 5596] dup_rows_per_dst=[2654, 2680, 2682, 2630, 2768, 2648, 2628, 2560] combine_rows_per_src=[5524, 5584, 5510, 5494, 5538, 5456, 5584, 5596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279638016 dispatch_meta_bytes=229568 combine_payload_bytes=279638016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279867584 ingress_bytes=389738496 link_bytes=389738496 token_bytes=559505600 weight_movement_bytes=110100480 total_nvlink_bytes=669606080 weight_to_token_ratio=0.196782 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778193455e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=229792 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279108000 ingress_bytes=499079168 link_bytes=499079168 token_bytes=557986208 weight_movement_bytes=220200960 total_nvlink_bytes=778187168 weight_to_token_ratio=0.394635 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.679807576e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=229680 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279107888 ingress_bytes=388978688 link_bytes=388978688 token_bytes=557986096 weight_movement_bytes=110100480 total_nvlink_bytes=668086576 weight_to_token_ratio=0.197318 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764979144e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=228856 combine_payload_bytes=279523328 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=279752184 ingress_bytes=830025728 link_bytes=830025728 token_bytes=559275512 weight_movement_bytes=550502400 total_nvlink_bytes=1109777912 weight_to_token_ratio=0.984313 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.443551257e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38922 dup_rows=21094 dup_rows_remote=18570 dedup_ratio=0.321869 router_tokens_per_expert=[528, 518, 511, 513, 527, 519, 494, 504, 494, 485, 514, 500, 515, 521, 525, 506, 502, 506, 460, 538, 530, 532, 478, 528, 530, 517, 486, 525, 499, 503, 503, 506, 500, 517, 485, 525, 529, 493, 497, 476, 523, 503, 516, 498, 509, 510, 512, 544, 522, 514, 531, 528, 521, 528, 542, 532, 498, 523, 505, 518, 468, 528, 509, 547] payload_rows_per_dst=[5530, 5528, 5478, 5498, 5520, 5626, 5682, 5580] dup_rows_per_dst=[2698, 2592, 2670, 2640, 2524, 2604, 2754, 2612] combine_rows_per_src=[5530, 5528, 5478, 5498, 5520, 5626, 5682, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229696 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279451968 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558674240 weight_movement_bytes=110100480 total_nvlink_bytes=668774720 weight_to_token_ratio=0.197075 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277846016 dispatch_meta_bytes=229432 combine_payload_bytes=277846016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278075448 ingress_bytes=498046976 link_bytes=498046976 token_bytes=555921464 weight_movement_bytes=220200960 total_nvlink_bytes=776122424 weight_to_token_ratio=0.396101 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661856059e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44492 payload_rows_remote=39014 dup_rows=21044 dup_rows_remote=18370 dedup_ratio=0.321106 router_tokens_per_expert=[546, 510, 510, 493, 514, 519, 593, 510, 480, 519, 487, 531, 514, 496, 497, 552, 523, 529, 525, 507, 506, 490, 537, 496, 543, 465, 539, 522, 477, 541, 477, 507, 484, 550, 501, 564, 506, 516, 479, 476, 515, 497, 494, 509, 504, 493, 515, 511, 535, 528, 489, 512, 490, 518, 502, 510, 498, 534, 524, 523, 507, 503, 527, 499] payload_rows_per_dst=[5708, 5478, 5596, 5544, 5574, 5504, 5500, 5588] dup_rows_per_dst=[2682, 2674, 2630, 2598, 2578, 2572, 2668, 2642] combine_rows_per_src=[5708, 5478, 5596, 5544, 5574, 5504, 5500, 5588] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44470 payload_rows_remote=38948 dup_rows=21066 dup_rows_remote=18382 dedup_ratio=0.321442 router_tokens_per_expert=[518, 530, 459, 503, 514, 523, 506, 522, 523, 516, 519, 552, 506, 503, 503, 517, 513, 508, 486, 507, 513, 538, 499, 509, 483, 508, 510, 508, 513, 499, 520, 483, 518, 514, 512, 539, 520, 516, 495, 544, 489, 459, 520, 535, 510, 462, 525, 547, 504, 483, 539, 516, 522, 529, 512, 498, 541, 492, 536, 535, 520, 459, 503, 563] payload_rows_per_dst=[5554, 5642, 5586, 5508, 5540, 5546, 5522, 5572] dup_rows_per_dst=[2596, 2636, 2560, 2540, 2776, 2548, 2684, 2726] combine_rows_per_src=[5554, 5642, 5586, 5508, 5540, 5546, 5522, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=229968 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279222864 ingress_bytes=609294336 link_bytes=609294336 token_bytes=558215760 weight_movement_bytes=330301440 total_nvlink_bytes=888517200 weight_to_token_ratio=0.591709 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059663062e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44310 payload_rows_remote=38788 dup_rows=21226 dup_rows_remote=18734 dedup_ratio=0.323883 router_tokens_per_expert=[520, 500, 518, 488, 508, 537, 527, 483, 508, 486, 522, 502, 508, 522, 494, 524, 512, 502, 499, 494, 477, 481, 546, 513, 546, 509, 493, 490, 498, 536, 520, 499, 502, 517, 496, 506, 512, 514, 511, 517, 477, 544, 486, 495, 466, 512, 519, 508, 535, 536, 500, 533, 510, 518, 539, 520, 506, 551, 517, 535, 529, 543, 514, 538] payload_rows_per_dst=[5546, 5474, 5440, 5560, 5566, 5522, 5552, 5650] dup_rows_per_dst=[2616, 2658, 2608, 2622, 2584, 2492, 2830, 2816] combine_rows_per_src=[5546, 5474, 5440, 5560, 5566, 5522, 5552, 5650] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38954 dup_rows=21034 dup_rows_remote=18388 dedup_ratio=0.320953 router_tokens_per_expert=[495, 502, 484, 518, 508, 521, 546, 528, 524, 499, 498, 545, 478, 542, 515, 495, 508, 528, 492, 512, 502, 486, 519, 516, 479, 513, 524, 534, 502, 540, 482, 523, 504, 523, 529, 517, 517, 536, 505, 522, 497, 485, 515, 519, 509, 506, 505, 520, 513, 510, 506, 520, 523, 535, 527, 507, 520, 530, 512, 484, 481, 530, 515, 488] payload_rows_per_dst=[5656, 5608, 5512, 5548, 5584, 5502, 5514, 5578] dup_rows_per_dst=[2548, 2584, 2614, 2646, 2722, 2610, 2768, 2542] combine_rows_per_src=[5656, 5608, 5512, 5548, 5584, 5502, 5514, 5578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=39014 dup_rows=21008 dup_rows_remote=18500 dedup_ratio=0.320557 router_tokens_per_expert=[541, 535, 517, 519, 472, 503, 506, 486, 534, 498, 496, 528, 520, 482, 474, 527, 493, 493, 498, 509, 535, 531, 514, 535, 488, 546, 504, 496, 539, 499, 513, 524, 520, 534, 533, 523, 536, 553, 496, 506, 517, 503, 521, 524, 479, 501, 511, 532, 556, 525, 504, 503, 482, 527, 508, 508, 479, 487, 513, 541, 513, 515, 488, 475] payload_rows_per_dst=[5384, 5476, 5568, 5662, 5710, 5612, 5602, 5514] dup_rows_per_dst=[2368, 2642, 2648, 2962, 2662, 2564, 2654, 2508] combine_rows_per_src=[5384, 5476, 5568, 5662, 5710, 5612, 5602, 5514] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38860 dup_rows=21168 dup_rows_remote=18532 dedup_ratio=0.322998 router_tokens_per_expert=[535, 528, 525, 483, 529, 521, 509, 517, 519, 484, 490, 492, 511, 475, 516, 489, 517, 448, 515, 523, 503, 531, 538, 497, 499, 499, 487, 530, 536, 507, 525, 506, 546, 508, 527, 509, 508, 512, 507, 513, 484, 513, 518, 508, 488, 499, 510, 515, 522, 471, 507, 522, 506, 544, 511, 510, 512, 532, 543, 542, 517, 527, 521, 532] payload_rows_per_dst=[5650, 5514, 5508, 5484, 5582, 5480, 5506, 5644] dup_rows_per_dst=[2644, 2438, 2636, 2694, 2678, 2590, 2680, 2808] combine_rows_per_src=[5650, 5514, 5508, 5484, 5582, 5480, 5506, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44528 payload_rows_remote=39014 dup_rows=21008 dup_rows_remote=18500 dedup_ratio=0.320557 router_tokens_per_expert=[541, 535, 517, 519, 472, 503, 506, 486, 534, 498, 496, 528, 520, 482, 474, 527, 493, 493, 498, 509, 535, 531, 514, 535, 488, 546, 504, 496, 539, 499, 513, 524, 520, 534, 533, 523, 536, 553, 496, 506, 517, 503, 521, 524, 479, 501, 511, 532, 556, 525, 504, 503, 482, 527, 508, 508, 479, 487, 513, 541, 513, 515, 488, 475] payload_rows_per_dst=[5384, 5476, 5568, 5662, 5710, 5612, 5602, 5514] dup_rows_per_dst=[2368, 2642, 2648, 2962, 2662, 2564, 2654, 2508] combine_rows_per_src=[5384, 5476, 5568, 5662, 5710, 5612, 5602, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44502 payload_rows_remote=38954 dup_rows=21034 dup_rows_remote=18388 dedup_ratio=0.320953 router_tokens_per_expert=[495, 502, 484, 518, 508, 521, 546, 528, 524, 499, 498, 545, 478, 542, 515, 495, 508, 528, 492, 512, 502, 486, 519, 516, 479, 513, 524, 534, 502, 540, 482, 523, 504, 523, 529, 517, 517, 536, 505, 522, 497, 485, 515, 519, 509, 506, 505, 520, 513, 510, 506, 520, 523, 535, 527, 507, 520, 530, 512, 484, 481, 530, 515, 488] payload_rows_per_dst=[5656, 5608, 5512, 5548, 5584, 5502, 5514, 5578] dup_rows_per_dst=[2548, 2584, 2614, 2646, 2722, 2610, 2768, 2542] combine_rows_per_src=[5656, 5608, 5512, 5548, 5584, 5502, 5514, 5578] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=229536 combine_payload_bytes=279652352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279881888 ingress_bytes=389752832 link_bytes=389752832 token_bytes=559534240 weight_movement_bytes=110100480 total_nvlink_bytes=669634720 weight_to_token_ratio=0.196772 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778442782e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44310 payload_rows_remote=38788 dup_rows=21226 dup_rows_remote=18734 dedup_ratio=0.323883 router_tokens_per_expert=[520, 500, 518, 488, 508, 537, 527, 483, 508, 486, 522, 502, 508, 522, 494, 524, 512, 502, 499, 494, 477, 481, 546, 513, 546, 509, 493, 490, 498, 536, 520, 499, 502, 517, 496, 506, 512, 514, 511, 517, 477, 544, 486, 495, 466, 512, 519, 508, 535, 536, 500, 533, 510, 518, 539, 520, 506, 551, 517, 535, 529, 543, 514, 538] payload_rows_per_dst=[5546, 5474, 5440, 5560, 5566, 5522, 5552, 5650] dup_rows_per_dst=[2616, 2658, 2608, 2622, 2584, 2492, 2830, 2816] combine_rows_per_src=[5546, 5474, 5440, 5560, 5566, 5522, 5552, 5650] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279179264 dispatch_meta_bytes=229320 combine_payload_bytes=279179264 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279408584 ingress_bytes=499380224 link_bytes=499380224 token_bytes=558587848 weight_movement_bytes=220200960 total_nvlink_bytes=778788808 weight_to_token_ratio=0.394210 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685043435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44368 payload_rows_remote=38860 dup_rows=21168 dup_rows_remote=18532 dedup_ratio=0.322998 router_tokens_per_expert=[535, 528, 525, 483, 529, 521, 509, 517, 519, 484, 490, 492, 511, 475, 516, 489, 517, 448, 515, 523, 503, 531, 538, 497, 499, 499, 487, 530, 536, 507, 525, 506, 546, 508, 527, 509, 508, 512, 507, 513, 484, 513, 518, 508, 488, 499, 510, 515, 522, 471, 507, 522, 506, 544, 511, 510, 512, 532, 543, 542, 517, 527, 521, 532] payload_rows_per_dst=[5650, 5514, 5508, 5484, 5582, 5480, 5506, 5644] dup_rows_per_dst=[2644, 2438, 2636, 2694, 2678, 2590, 2680, 2808] combine_rows_per_src=[5650, 5514, 5508, 5484, 5582, 5480, 5506, 5644] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229368 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279451640 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558673912 weight_movement_bytes=110100480 total_nvlink_bytes=668774392 weight_to_token_ratio=0.197075 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=230056 combine_payload_bytes=279652352 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279882408 ingress_bytes=609953792 link_bytes=609953792 token_bytes=559534760 weight_movement_bytes=330301440 total_nvlink_bytes=889836200 weight_to_token_ratio=0.590314 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060809964e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44408 payload_rows_remote=38762 dup_rows=21128 dup_rows_remote=18418 dedup_ratio=0.322388 router_tokens_per_expert=[502, 515, 509, 527, 486, 557, 538, 507, 515, 486, 520, 494, 562, 534, 481, 496, 498, 519, 500, 514, 527, 545, 484, 531, 495, 477, 496, 525, 479, 520, 535, 488, 483, 524, 477, 512, 513, 514, 522, 544, 514, 499, 521, 491, 519, 496, 510, 527, 514, 496, 513, 578, 499, 493, 518, 504, 536, 514, 536, 488, 542, 501, 493, 515] payload_rows_per_dst=[5646, 5746, 5610, 5474, 5494, 5782, 5216, 5440] dup_rows_per_dst=[2710, 2846, 2872, 2556, 2442, 2982, 2404, 2316] combine_rows_per_src=[5646, 5746, 5610, 5474, 5494, 5782, 5216, 5440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44334 payload_rows_remote=38686 dup_rows=21202 dup_rows_remote=18408 dedup_ratio=0.323517 router_tokens_per_expert=[499, 493, 539, 523, 463, 506, 537, 514, 507, 499, 495, 498, 509, 506, 531, 514, 541, 529, 498, 518, 478, 470, 523, 490, 518, 524, 501, 502, 478, 535, 499, 492, 514, 568, 504, 528, 537, 554, 475, 541, 499, 525, 476, 551, 535, 542, 568, 527, 518, 509, 535, 450, 475, 530, 507, 534, 461, 520, 506, 505, 469, 538, 532, 506] payload_rows_per_dst=[5558, 5550, 5490, 5532, 5648, 5602, 5482, 5472] dup_rows_per_dst=[2590, 2568, 2604, 2566, 2794, 2844, 2634, 2602] combine_rows_per_src=[5558, 5550, 5490, 5532, 5648, 5602, 5482, 5472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44442 payload_rows_remote=38922 dup_rows=21094 dup_rows_remote=18570 dedup_ratio=0.321869 router_tokens_per_expert=[528, 518, 511, 513, 527, 519, 494, 504, 494, 485, 514, 500, 515, 521, 525, 506, 502, 506, 460, 538, 530, 532, 478, 528, 530, 517, 486, 525, 499, 503, 503, 506, 500, 517, 485, 525, 529, 493, 497, 476, 523, 503, 516, 498, 509, 510, 512, 544, 522, 514, 531, 528, 521, 528, 542, 532, 498, 523, 505, 518, 468, 528, 509, 547] payload_rows_per_dst=[5530, 5528, 5478, 5498, 5520, 5626, 5682, 5580] dup_rows_per_dst=[2698, 2592, 2670, 2640, 2524, 2604, 2754, 2612] combine_rows_per_src=[5530, 5528, 5478, 5498, 5520, 5626, 5682, 5580] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278032384 dispatch_meta_bytes=230088 combine_payload_bytes=278032384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278262472 ingress_bytes=388132864 link_bytes=388132864 token_bytes=556294856 weight_movement_bytes=110100480 total_nvlink_bytes=666395336 weight_to_token_ratio=0.197917 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750268874e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44408 payload_rows_remote=38762 dup_rows=21128 dup_rows_remote=18418 dedup_ratio=0.322388 router_tokens_per_expert=[502, 515, 509, 527, 486, 557, 538, 507, 515, 486, 520, 494, 562, 534, 481, 496, 498, 519, 500, 514, 527, 545, 484, 531, 495, 477, 496, 525, 479, 520, 535, 488, 483, 524, 477, 512, 513, 514, 522, 544, 514, 499, 521, 491, 519, 496, 510, 527, 514, 496, 513, 578, 499, 493, 518, 504, 536, 514, 536, 488, 542, 501, 493, 515] payload_rows_per_dst=[5646, 5746, 5610, 5474, 5494, 5782, 5216, 5440] dup_rows_per_dst=[2710, 2846, 2872, 2556, 2442, 2982, 2404, 2316] combine_rows_per_src=[5646, 5746, 5610, 5474, 5494, 5782, 5216, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278548480 dispatch_meta_bytes=229568 combine_payload_bytes=278548480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278778048 ingress_bytes=388648960 link_bytes=388648960 token_bytes=557326528 weight_movement_bytes=110100480 total_nvlink_bytes=667427008 weight_to_token_ratio=0.197551 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759244632e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=230056 combine_payload_bytes=279652352 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279882408 ingress_bytes=609953792 link_bytes=609953792 token_bytes=559534760 weight_movement_bytes=330301440 total_nvlink_bytes=889836200 weight_to_token_ratio=0.590314 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060809964e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279222272 dispatch_meta_bytes=229368 combine_payload_bytes=279222272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279451640 ingress_bytes=389322752 link_bytes=389322752 token_bytes=558673912 weight_movement_bytes=110100480 total_nvlink_bytes=668774392 weight_to_token_ratio=0.197075 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.770962983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278992896 dispatch_meta_bytes=229968 combine_payload_bytes=278992896 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279222864 ingress_bytes=609294336 link_bytes=609294336 token_bytes=558215760 weight_movement_bytes=330301440 total_nvlink_bytes=888517200 weight_to_token_ratio=0.591709 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059663062e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277846016 dispatch_meta_bytes=228720 combine_payload_bytes=277846016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278074736 ingress_bytes=498046976 link_bytes=498046976 token_bytes=555920752 weight_movement_bytes=220200960 total_nvlink_bytes=776121712 weight_to_token_ratio=0.396101 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661856059e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44398 payload_rows_remote=38842 dup_rows=21138 dup_rows_remote=18436 dedup_ratio=0.322540 router_tokens_per_expert=[481, 517, 504, 488, 534, 543, 490, 523, 531, 505, 511, 533, 503, 496, 498, 552, 529, 519, 498, 534, 521, 514, 512, 494, 486, 568, 525, 520, 501, 537, 522, 482, 535, 516, 520, 502, 474, 495, 489, 515, 487, 543, 507, 486, 515, 527, 501, 488, 511, 517, 496, 508, 523, 479, 491, 546, 498, 514, 508, 514, 534, 517, 527, 514] payload_rows_per_dst=[5576, 5556, 5562, 5616, 5496, 5506, 5530, 5556] dup_rows_per_dst=[2584, 2702, 2680, 2666, 2596, 2602, 2612, 2696] combine_rows_per_src=[5576, 5556, 5562, 5616, 5496, 5506, 5530, 5556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277301248 dispatch_meta_bytes=228376 combine_payload_bytes=277301248 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277529624 ingress_bytes=497502208 link_bytes=497502208 token_bytes=554830872 weight_movement_bytes=220200960 total_nvlink_bytes=775031832 weight_to_token_ratio=0.396879 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652381648e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38834 dup_rows=21088 dup_rows_remote=18518 dedup_ratio=0.321777 router_tokens_per_expert=[544, 533, 512, 490, 545, 486, 523, 549, 516, 497, 532, 489, 545, 507, 504, 499, 476, 497, 487, 499, 502, 509, 465, 502, 497, 501, 530, 559, 498, 496, 523, 511, 516, 538, 480, 522, 538, 545, 505, 495, 522, 504, 485, 522, 495, 484, 548, 517, 506, 537, 483, 507, 495, 505, 558, 501, 542, 497, 517, 493, 522, 501, 549, 516] payload_rows_per_dst=[5596, 5550, 5406, 5572, 5600, 5508, 5614, 5602] dup_rows_per_dst=[2768, 2628, 2468, 2658, 2678, 2646, 2570, 2672] combine_rows_per_src=[5596, 5550, 5406, 5572, 5600, 5508, 5614, 5602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44308 payload_rows_remote=38726 dup_rows=21228 dup_rows_remote=18614 dedup_ratio=0.323914 router_tokens_per_expert=[503, 481, 503, 495, 522, 493, 509, 513, 527, 496, 551, 547, 503, 483, 547, 508, 533, 507, 508, 517, 479, 498, 546, 542, 560, 494, 481, 528, 542, 499, 491, 506, 561, 507, 508, 495, 511, 483, 507, 512, 517, 522, 468, 529, 499, 468, 520, 504, 518, 535, 556, 536, 518, 501, 502, 491, 505, 501, 533, 496, 515, 513, 534, 491] payload_rows_per_dst=[5418, 5558, 5596, 5542, 5356, 5520, 5736, 5582] dup_rows_per_dst=[2440, 2766, 2782, 2640, 2420, 2596, 2970, 2614] combine_rows_per_src=[5418, 5558, 5596, 5542, 5356, 5520, 5736, 5582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38952 dup_rows=20966 dup_rows_remote=18368 dedup_ratio=0.319916 router_tokens_per_expert=[555, 545, 472, 510, 565, 521, 495, 502, 529, 507, 509, 479, 508, 545, 494, 519, 484, 531, 527, 479, 517, 537, 501, 487, 553, 503, 526, 514, 524, 497, 507, 484, 510, 549, 532, 539, 514, 484, 524, 511, 518, 512, 570, 514, 494, 472, 487, 522, 495, 554, 484, 499, 519, 459, 483, 525, 503, 528, 469, 530, 491, 500, 550, 501] payload_rows_per_dst=[5570, 5560, 5556, 5618, 5678, 5548, 5476, 5564] dup_rows_per_dst=[2760, 2620, 2570, 2598, 2648, 2630, 2560, 2580] combine_rows_per_src=[5570, 5560, 5556, 5618, 5678, 5548, 5476, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38864 dup_rows=21092 dup_rows_remote=18466 dedup_ratio=0.321838 router_tokens_per_expert=[469, 504, 518, 537, 495, 517, 542, 545, 519, 487, 518, 504, 553, 517, 519, 522, 527, 512, 488, 501, 528, 522, 526, 546, 500, 534, 476, 512, 523, 512, 509, 508, 496, 528, 488, 489, 531, 506, 538, 497, 491, 495, 538, 494, 533, 489, 543, 520, 476, 524, 486, 487, 463, 530, 533, 499, 469, 523, 522, 512, 521, 502, 556, 499] payload_rows_per_dst=[5596, 5612, 5502, 5520, 5520, 5580, 5516, 5598] dup_rows_per_dst=[2658, 2666, 2798, 2628, 2626, 2626, 2480, 2610] combine_rows_per_src=[5596, 5612, 5502, 5520, 5520, 5580, 5516, 5598] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44496 payload_rows_remote=38910 dup_rows=21040 dup_rows_remote=18378 dedup_ratio=0.321045 router_tokens_per_expert=[494, 486, 502, 526, 526, 516, 514, 505, 522, 506, 503, 525, 464, 526, 529, 489, 498, 517, 520, 514, 531, 544, 489, 511, 514, 491, 515, 501, 536, 535, 542, 487, 499, 482, 519, 512, 495, 514, 522, 522, 495, 507, 514, 512, 531, 464, 577, 503, 518, 470, 476, 538, 504, 515, 526, 522, 535, 499, 531, 570, 483, 502, 511, 522] payload_rows_per_dst=[5488, 5540, 5586, 5590, 5570, 5566, 5552, 5604] dup_rows_per_dst=[2650, 2588, 2662, 2652, 2560, 2640, 2586, 2702] combine_rows_per_src=[5488, 5540, 5586, 5590, 5570, 5566, 5552, 5604] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=229112 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278648568 ingress_bytes=498620416 link_bytes=498620416 token_bytes=557068024 weight_movement_bytes=220200960 total_nvlink_bytes=777268984 weight_to_token_ratio=0.395286 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671829124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44308 payload_rows_remote=38726 dup_rows=21228 dup_rows_remote=18614 dedup_ratio=0.323914 router_tokens_per_expert=[503, 481, 503, 495, 522, 493, 509, 513, 527, 496, 551, 547, 503, 483, 547, 508, 533, 507, 508, 517, 479, 498, 546, 542, 560, 494, 481, 528, 542, 499, 491, 506, 561, 507, 508, 495, 511, 483, 507, 512, 517, 522, 468, 529, 499, 468, 520, 504, 518, 535, 556, 536, 518, 501, 502, 491, 505, 501, 533, 496, 515, 513, 534, 491] payload_rows_per_dst=[5418, 5558, 5596, 5542, 5356, 5520, 5736, 5582] dup_rows_per_dst=[2440, 2766, 2782, 2640, 2420, 2596, 2970, 2614] combine_rows_per_src=[5418, 5558, 5596, 5542, 5356, 5520, 5736, 5582] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278362112 dispatch_meta_bytes=229408 combine_payload_bytes=278362112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278591520 ingress_bytes=388462592 link_bytes=388462592 token_bytes=556953632 weight_movement_bytes=110100480 total_nvlink_bytes=667054112 weight_to_token_ratio=0.197683 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.756003386e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277587968 dispatch_meta_bytes=229360 combine_payload_bytes=277587968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277817328 ingress_bytes=497788928 link_bytes=497788928 token_bytes=555405296 weight_movement_bytes=220200960 total_nvlink_bytes=775606256 weight_to_token_ratio=0.396469 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.657368180e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44496 payload_rows_remote=38910 dup_rows=21040 dup_rows_remote=18378 dedup_ratio=0.321045 router_tokens_per_expert=[494, 486, 502, 526, 526, 516, 514, 505, 522, 506, 503, 525, 464, 526, 529, 489, 498, 517, 520, 514, 531, 544, 489, 511, 514, 491, 515, 501, 536, 535, 542, 487, 499, 482, 519, 512, 495, 514, 522, 522, 495, 507, 514, 512, 531, 464, 577, 503, 518, 470, 476, 538, 504, 515, 526, 522, 535, 499, 531, 570, 483, 502, 511, 522] payload_rows_per_dst=[5488, 5540, 5586, 5590, 5570, 5566, 5552, 5604] dup_rows_per_dst=[2650, 2588, 2662, 2652, 2560, 2640, 2586, 2702] combine_rows_per_src=[5488, 5540, 5586, 5590, 5570, 5566, 5552, 5604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44444 payload_rows_remote=38864 dup_rows=21092 dup_rows_remote=18466 dedup_ratio=0.321838 router_tokens_per_expert=[469, 504, 518, 537, 495, 517, 542, 545, 519, 487, 518, 504, 553, 517, 519, 522, 527, 512, 488, 501, 528, 522, 526, 546, 500, 534, 476, 512, 523, 512, 509, 508, 496, 528, 488, 489, 531, 506, 538, 497, 491, 495, 538, 494, 533, 489, 543, 520, 476, 524, 486, 487, 463, 530, 533, 499, 469, 523, 522, 512, 521, 502, 556, 499] payload_rows_per_dst=[5596, 5612, 5502, 5520, 5520, 5580, 5516, 5598] dup_rows_per_dst=[2658, 2666, 2798, 2628, 2626, 2626, 2480, 2610] combine_rows_per_src=[5596, 5612, 5502, 5520, 5520, 5580, 5516, 5598] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229280 combine_payload_bytes=279207936 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279437216 ingress_bytes=499408896 link_bytes=499408896 token_bytes=558645152 weight_movement_bytes=220200960 total_nvlink_bytes=778846112 weight_to_token_ratio=0.394170 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685542088e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38896 dup_rows=21050 dup_rows_remote=18356 dedup_ratio=0.321198 router_tokens_per_expert=[495, 565, 476, 534, 524, 478, 549, 507, 467, 497, 521, 511, 536, 515, 478, 523, 508, 521, 509, 529, 513, 506, 536, 515, 513, 507, 515, 473, 498, 499, 545, 521, 523, 491, 531, 490, 481, 516, 500, 542, 503, 532, 493, 510, 496, 516, 516, 495, 538, 533, 496, 518, 507, 504, 510, 503, 516, 485, 547, 531, 524, 549, 486, 502] payload_rows_per_dst=[5590, 5460, 5638, 5536, 5628, 5530, 5568, 5536] dup_rows_per_dst=[2694, 2612, 2636, 2602, 2718, 2592, 2650, 2546] combine_rows_per_src=[5590, 5460, 5638, 5536, 5628, 5530, 5568, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229320 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278806472 ingress_bytes=388677632 link_bytes=388677632 token_bytes=557383624 weight_movement_bytes=110100480 total_nvlink_bytes=667484104 weight_to_token_ratio=0.197531 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759743285e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44570 payload_rows_remote=38952 dup_rows=20966 dup_rows_remote=18368 dedup_ratio=0.319916 router_tokens_per_expert=[555, 545, 472, 510, 565, 521, 495, 502, 529, 507, 509, 479, 508, 545, 494, 519, 484, 531, 527, 479, 517, 537, 501, 487, 553, 503, 526, 514, 524, 497, 507, 484, 510, 549, 532, 539, 514, 484, 524, 511, 518, 512, 570, 514, 494, 472, 487, 522, 495, 554, 484, 499, 519, 459, 483, 525, 503, 528, 469, 530, 491, 500, 550, 501] payload_rows_per_dst=[5570, 5560, 5556, 5618, 5678, 5548, 5476, 5564] dup_rows_per_dst=[2760, 2620, 2570, 2598, 2648, 2630, 2560, 2580] combine_rows_per_src=[5570, 5560, 5556, 5618, 5678, 5548, 5476, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44334 payload_rows_remote=38686 dup_rows=21202 dup_rows_remote=18408 dedup_ratio=0.323517 router_tokens_per_expert=[499, 493, 539, 523, 463, 506, 537, 514, 507, 499, 495, 498, 509, 506, 531, 514, 541, 529, 498, 518, 478, 470, 523, 490, 518, 524, 501, 502, 478, 535, 499, 492, 514, 568, 504, 528, 537, 554, 475, 541, 499, 525, 476, 551, 535, 542, 568, 527, 518, 509, 535, 450, 475, 530, 507, 534, 461, 520, 506, 505, 469, 538, 532, 506] payload_rows_per_dst=[5558, 5550, 5490, 5532, 5648, 5602, 5482, 5472] dup_rows_per_dst=[2590, 2568, 2604, 2566, 2794, 2844, 2634, 2602] combine_rows_per_src=[5558, 5550, 5490, 5532, 5648, 5602, 5482, 5472] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38868 dup_rows=21150 dup_rows_remote=18546 dedup_ratio=0.322723 router_tokens_per_expert=[507, 523, 489, 487, 552, 515, 526, 483, 496, 525, 506, 507, 513, 503, 526, 529, 510, 550, 499, 592, 525, 525, 496, 529, 531, 480, 496, 492, 533, 461, 507, 526, 555, 487, 525, 474, 510, 518, 487, 505, 493, 496, 494, 477, 525, 542, 494, 535, 518, 538, 507, 479, 546, 513, 487, 508, 515, 510, 538, 513, 513, 521, 514, 492] payload_rows_per_dst=[5562, 5556, 5642, 5450, 5518, 5492, 5586, 5580] dup_rows_per_dst=[2602, 2654, 2810, 2602, 2604, 2620, 2606, 2652] combine_rows_per_src=[5562, 5556, 5642, 5450, 5518, 5492, 5586, 5580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44486 payload_rows_remote=38896 dup_rows=21050 dup_rows_remote=18356 dedup_ratio=0.321198 router_tokens_per_expert=[495, 565, 476, 534, 524, 478, 549, 507, 467, 497, 521, 511, 536, 515, 478, 523, 508, 521, 509, 529, 513, 506, 536, 515, 513, 507, 515, 473, 498, 499, 545, 521, 523, 491, 531, 490, 481, 516, 500, 542, 503, 532, 493, 510, 496, 516, 516, 495, 538, 533, 496, 518, 507, 504, 510, 503, 516, 485, 547, 531, 524, 549, 486, 502] payload_rows_per_dst=[5590, 5460, 5638, 5536, 5628, 5530, 5568, 5536] dup_rows_per_dst=[2694, 2612, 2636, 2602, 2718, 2592, 2650, 2546] combine_rows_per_src=[5590, 5460, 5638, 5536, 5628, 5530, 5568, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278906880 dispatch_meta_bytes=229152 combine_payload_bytes=278906880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279136032 ingress_bytes=389007360 link_bytes=389007360 token_bytes=558042912 weight_movement_bytes=110100480 total_nvlink_bytes=668143392 weight_to_token_ratio=0.197298 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765477798e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229320 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278806472 ingress_bytes=388677632 link_bytes=388677632 token_bytes=557383624 weight_movement_bytes=110100480 total_nvlink_bytes=667484104 weight_to_token_ratio=0.197531 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.759743285e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=229008 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279035536 ingress_bytes=609107968 link_bytes=609107968 token_bytes=557842064 weight_movement_bytes=330301440 total_nvlink_bytes=888143504 weight_to_token_ratio=0.592106 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059338937e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277587968 dispatch_meta_bytes=229360 combine_payload_bytes=277587968 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277817328 ingress_bytes=497788928 link_bytes=497788928 token_bytes=555405296 weight_movement_bytes=220200960 total_nvlink_bytes=775606256 weight_to_token_ratio=0.396469 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.657368180e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277301248 dispatch_meta_bytes=228376 combine_payload_bytes=277301248 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277529624 ingress_bytes=497502208 link_bytes=497502208 token_bytes=554830872 weight_movement_bytes=220200960 total_nvlink_bytes=775031832 weight_to_token_ratio=0.396879 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652381648e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279207936 dispatch_meta_bytes=229280 combine_payload_bytes=279207936 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=279437216 ingress_bytes=499408896 link_bytes=499408896 token_bytes=558645152 weight_movement_bytes=220200960 total_nvlink_bytes=778846112 weight_to_token_ratio=0.394170 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.685542088e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44616 payload_rows_remote=39056 dup_rows=20920 dup_rows_remote=18292 dedup_ratio=0.319214 router_tokens_per_expert=[512, 548, 496, 491, 521, 547, 475, 531, 531, 521, 527, 479, 497, 535, 501, 503, 542, 469, 511, 519, 514, 564, 514, 532, 478, 493, 501, 498, 504, 497, 521, 524, 509, 532, 509, 507, 511, 485, 522, 532, 456, 532, 513, 529, 521, 523, 495, 481, 516, 507, 498, 509, 515, 576, 505, 526, 519, 515, 507, 482, 487, 506, 509, 538] payload_rows_per_dst=[5578, 5560, 5650, 5504, 5552, 5562, 5646, 5564] dup_rows_per_dst=[2664, 2628, 2680, 2528, 2662, 2538, 2658, 2562] combine_rows_per_src=[5578, 5560, 5650, 5504, 5552, 5562, 5646, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=229656 combine_payload_bytes=278605824 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=278835480 ingress_bytes=388706304 link_bytes=388706304 token_bytes=557441304 weight_movement_bytes=110100480 total_nvlink_bytes=667541784 weight_to_token_ratio=0.197510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760241939e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44458 payload_rows_remote=38692 dup_rows=21078 dup_rows_remote=18108 dedup_ratio=0.321625 router_tokens_per_expert=[497, 488, 532, 478, 488, 500, 545, 492, 506, 477, 493, 524, 551, 514, 541, 510, 480, 514, 501, 547, 505, 485, 496, 487, 539, 528, 517, 525, 498, 501, 515, 489, 519, 488, 526, 505, 524, 515, 517, 499, 528, 490, 553, 514, 483, 530, 515, 489, 502, 527, 513, 509, 509, 533, 531, 522, 543, 520, 535, 541, 499, 529, 521, 476] payload_rows_per_dst=[5514, 5430, 5194, 5816, 5542, 5696, 5500, 5766] dup_rows_per_dst=[2526, 2366, 2158, 3044, 2644, 2944, 2426, 2970] combine_rows_per_src=[5514, 5430, 5194, 5816, 5542, 5696, 5500, 5766] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278806528 dispatch_meta_bytes=229008 combine_payload_bytes=278806528 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279035536 ingress_bytes=609107968 link_bytes=609107968 token_bytes=557842064 weight_movement_bytes=330301440 total_nvlink_bytes=888143504 weight_to_token_ratio=0.592106 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059338937e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=38990 dup_rows=20946 dup_rows_remote=18222 dedup_ratio=0.319611 router_tokens_per_expert=[523, 483, 482, 523, 552, 451, 527, 505, 531, 525, 509, 503, 497, 490, 512, 494, 523, 497, 501, 541, 536, 568, 508, 504, 521, 534, 501, 530, 468, 500, 535, 516, 498, 511, 504, 490, 519, 530, 533, 536, 520, 520, 514, 497, 489, 501, 525, 513, 475, 518, 540, 541, 539, 508, 555, 486, 503, 512, 499, 538, 470, 502, 490, 502] payload_rows_per_dst=[5604, 5494, 5666, 5528, 5622, 5550, 5600, 5526] dup_rows_per_dst=[2488, 2628, 2690, 2682, 2620, 2608, 2724, 2506] combine_rows_per_src=[5604, 5494, 5666, 5528, 5622, 5550, 5600, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38984 dup_rows=21130 dup_rows_remote=18644 dedup_ratio=0.322418 router_tokens_per_expert=[524, 482, 527, 520, 489, 493, 508, 500, 500, 497, 524, 561, 526, 508, 546, 549, 489, 523, 528, 539, 505, 477, 481, 493, 483, 465, 542, 553, 522, 474, 517, 508, 531, 550, 513, 517, 548, 534, 463, 544, 505, 508, 524, 496, 490, 478, 472, 481, 534, 535, 518, 517, 542, 523, 516, 527, 516, 514, 513, 479, 509, 534, 465, 519] payload_rows_per_dst=[5446, 5654, 5540, 5458, 5664, 5422, 5716, 5506] dup_rows_per_dst=[2640, 2768, 2530, 2670, 2736, 2486, 2708, 2592] combine_rows_per_src=[5446, 5654, 5540, 5458, 5664, 5422, 5716, 5506] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38924 dup_rows=21072 dup_rows_remote=18466 dedup_ratio=0.321533 router_tokens_per_expert=[551, 477, 551, 523, 490, 503, 503, 475, 523, 489, 514, 478, 522, 503, 506, 525, 512, 500, 506, 527, 523, 511, 514, 545, 498, 519, 508, 503, 517, 520, 527, 481, 517, 517, 498, 483, 539, 548, 517, 478, 528, 512, 488, 489, 508, 543, 518, 515, 563, 555, 551, 533, 525, 497, 510, 491, 506, 486, 509, 511, 460, 526, 507, 496] payload_rows_per_dst=[5498, 5502, 5646, 5540, 5550, 5510, 5656, 5562] dup_rows_per_dst=[2648, 2618, 2630, 2606, 2644, 2692, 2794, 2440] combine_rows_per_src=[5498, 5502, 5646, 5540, 5550, 5510, 5656, 5562] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279953408 dispatch_meta_bytes=229392 combine_payload_bytes=279953408 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280182800 ingress_bytes=500154368 link_bytes=500154368 token_bytes=560136208 weight_movement_bytes=220200960 total_nvlink_bytes=780337168 weight_to_token_ratio=0.393120 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.698507072e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277344256 dispatch_meta_bytes=227200 combine_payload_bytes=277344256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277571456 ingress_bytes=387444736 link_bytes=387444736 token_bytes=554915712 weight_movement_bytes=110100480 total_nvlink_bytes=665016192 weight_to_token_ratio=0.198409 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.738301196e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44536 payload_rows_remote=38924 dup_rows=21000 dup_rows_remote=18220 dedup_ratio=0.320435 router_tokens_per_expert=[515, 549, 478, 527, 485, 500, 519, 513, 522, 503, 540, 506, 513, 509, 525, 490, 554, 535, 512, 504, 503, 545, 506, 537, 518, 485, 516, 515, 510, 497, 488, 504, 508, 500, 492, 523, 499, 503, 526, 506, 510, 520, 497, 472, 523, 514, 522, 506, 506, 502, 507, 564, 539, 535, 490, 520, 546, 498, 492, 538, 479, 498, 505, 505] payload_rows_per_dst=[5606, 5468, 5612, 5552, 5528, 5568, 5666, 5536] dup_rows_per_dst=[2566, 2748, 2780, 2514, 2586, 2560, 2660, 2586] combine_rows_per_src=[5606, 5468, 5612, 5552, 5528, 5568, 5666, 5536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=230512 combine_payload_bytes=279437312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279667824 ingress_bytes=389537792 link_bytes=389537792 token_bytes=559105136 weight_movement_bytes=110100480 total_nvlink_bytes=669205616 weight_to_token_ratio=0.196923 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774702882e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279480320 dispatch_meta_bytes=228848 combine_payload_bytes=279480320 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279709168 ingress_bytes=609781760 link_bytes=609781760 token_bytes=559189488 weight_movement_bytes=330301440 total_nvlink_bytes=889490928 weight_to_token_ratio=0.590679 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060510773e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44406 payload_rows_remote=38984 dup_rows=21130 dup_rows_remote=18644 dedup_ratio=0.322418 router_tokens_per_expert=[524, 482, 527, 520, 489, 493, 508, 500, 500, 497, 524, 561, 526, 508, 546, 549, 489, 523, 528, 539, 505, 477, 481, 493, 483, 465, 542, 553, 522, 474, 517, 508, 531, 550, 513, 517, 548, 534, 463, 544, 505, 508, 524, 496, 490, 478, 472, 481, 534, 535, 518, 517, 542, 523, 516, 527, 516, 514, 513, 479, 509, 534, 465, 519] payload_rows_per_dst=[5446, 5654, 5540, 5458, 5664, 5422, 5716, 5506] dup_rows_per_dst=[2640, 2768, 2530, 2670, 2736, 2486, 2708, 2592] combine_rows_per_src=[5446, 5654, 5540, 5458, 5664, 5422, 5716, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38720 dup_rows=21052 dup_rows_remote=18222 dedup_ratio=0.321228 router_tokens_per_expert=[515, 510, 472, 491, 492, 498, 492, 491, 519, 558, 487, 511, 521, 532, 557, 528, 574, 526, 528, 511, 492, 476, 506, 552, 525, 541, 466, 520, 516, 475, 469, 509, 488, 522, 524, 524, 481, 520, 464, 516, 525, 543, 531, 501, 522, 506, 502, 514, 483, 535, 535, 525, 522, 517, 525, 489, 514, 503, 509, 543, 500, 494, 518, 513] payload_rows_per_dst=[5764, 5608, 5558, 5502, 5466, 5644, 5356, 5586] dup_rows_per_dst=[2830, 2854, 2750, 2540, 2598, 2644, 2234, 2602] combine_rows_per_src=[5764, 5608, 5558, 5502, 5466, 5644, 5356, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44458 payload_rows_remote=38692 dup_rows=21078 dup_rows_remote=18108 dedup_ratio=0.321625 router_tokens_per_expert=[497, 488, 532, 478, 488, 500, 545, 492, 506, 477, 493, 524, 551, 514, 541, 510, 480, 514, 501, 547, 505, 485, 496, 487, 539, 528, 517, 525, 498, 501, 515, 489, 519, 488, 526, 505, 524, 515, 517, 499, 528, 490, 553, 514, 483, 530, 515, 489, 502, 527, 513, 509, 509, 533, 531, 522, 543, 520, 535, 541, 499, 529, 521, 476] payload_rows_per_dst=[5514, 5430, 5194, 5816, 5542, 5696, 5500, 5766] dup_rows_per_dst=[2526, 2366, 2158, 3044, 2644, 2944, 2426, 2970] combine_rows_per_src=[5514, 5430, 5194, 5816, 5542, 5696, 5500, 5766] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44386 payload_rows_remote=38868 dup_rows=21150 dup_rows_remote=18546 dedup_ratio=0.322723 router_tokens_per_expert=[507, 523, 489, 487, 552, 515, 526, 483, 496, 525, 506, 507, 513, 503, 526, 529, 510, 550, 499, 592, 525, 525, 496, 529, 531, 480, 496, 492, 533, 461, 507, 526, 555, 487, 525, 474, 510, 518, 487, 505, 493, 496, 494, 477, 525, 542, 494, 535, 518, 538, 507, 479, 546, 513, 487, 508, 515, 510, 538, 513, 513, 521, 514, 492] payload_rows_per_dst=[5562, 5556, 5642, 5450, 5518, 5492, 5586, 5580] dup_rows_per_dst=[2602, 2654, 2810, 2602, 2604, 2620, 2606, 2652] combine_rows_per_src=[5562, 5556, 5642, 5450, 5518, 5492, 5586, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38924 dup_rows=21072 dup_rows_remote=18466 dedup_ratio=0.321533 router_tokens_per_expert=[551, 477, 551, 523, 490, 503, 503, 475, 523, 489, 514, 478, 522, 503, 506, 525, 512, 500, 506, 527, 523, 511, 514, 545, 498, 519, 508, 503, 517, 520, 527, 481, 517, 517, 498, 483, 539, 548, 517, 478, 528, 512, 488, 489, 508, 543, 518, 515, 563, 555, 551, 533, 525, 497, 510, 491, 506, 486, 509, 511, 460, 526, 507, 496] payload_rows_per_dst=[5498, 5502, 5646, 5540, 5550, 5510, 5656, 5562] dup_rows_per_dst=[2648, 2618, 2630, 2606, 2644, 2692, 2794, 2440] combine_rows_per_src=[5498, 5502, 5646, 5540, 5550, 5510, 5656, 5562] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229560 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279236792 ingress_bytes=389107712 link_bytes=389107712 token_bytes=558244024 weight_movement_bytes=110100480 total_nvlink_bytes=668344504 weight_to_token_ratio=0.197226 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767223084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44304 payload_rows_remote=38708 dup_rows=21232 dup_rows_remote=18530 dedup_ratio=0.323975 router_tokens_per_expert=[485, 482, 533, 485, 509, 471, 433, 507, 496, 539, 526, 526, 511, 477, 563, 486, 528, 499, 490, 520, 510, 546, 538, 476, 503, 524, 521, 552, 536, 520, 474, 513, 539, 522, 490, 511, 492, 559, 523, 513, 526, 494, 488, 489, 495, 485, 516, 508, 516, 518, 537, 539, 491, 540, 519, 534, 565, 514, 495, 500, 501, 523, 539, 508] payload_rows_per_dst=[5450, 5520, 5574, 5612, 5596, 5440, 5568, 5544] dup_rows_per_dst=[2360, 2728, 2640, 2674, 2702, 2562, 2820, 2746] combine_rows_per_src=[5450, 5520, 5574, 5612, 5596, 5440, 5568, 5544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=38720 dup_rows=21052 dup_rows_remote=18222 dedup_ratio=0.321228 router_tokens_per_expert=[515, 510, 472, 491, 492, 498, 492, 491, 519, 558, 487, 511, 521, 532, 557, 528, 574, 526, 528, 511, 492, 476, 506, 552, 525, 541, 466, 520, 516, 475, 469, 509, 488, 522, 524, 524, 481, 520, 464, 516, 525, 543, 531, 501, 522, 506, 502, 514, 483, 535, 535, 525, 522, 517, 525, 489, 514, 503, 509, 543, 500, 494, 518, 513] payload_rows_per_dst=[5764, 5608, 5558, 5502, 5466, 5644, 5356, 5586] dup_rows_per_dst=[2830, 2854, 2750, 2540, 2598, 2644, 2234, 2602] combine_rows_per_src=[5764, 5608, 5558, 5502, 5466, 5644, 5356, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=228576 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=279235808 ingress_bytes=719409152 link_bytes=719409152 token_bytes=558243040 weight_movement_bytes=440401920 total_nvlink_bytes=998644960 weight_to_token_ratio=0.788907 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.251170838e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=230512 combine_payload_bytes=279437312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279667824 ingress_bytes=389537792 link_bytes=389537792 token_bytes=559105136 weight_movement_bytes=110100480 total_nvlink_bytes=669205616 weight_to_token_ratio=0.196923 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.774702882e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277544960 dispatch_meta_bytes=227768 combine_payload_bytes=277544960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277772728 ingress_bytes=387645440 link_bytes=387645440 token_bytes=555317688 weight_movement_bytes=110100480 total_nvlink_bytes=665418168 weight_to_token_ratio=0.198266 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.741791769e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277458944 dispatch_meta_bytes=228952 combine_payload_bytes=277458944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277687896 ingress_bytes=387559424 link_bytes=387559424 token_bytes=555146840 weight_movement_bytes=110100480 total_nvlink_bytes=665247320 weight_to_token_ratio=0.198327 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.740295809e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277344256 dispatch_meta_bytes=227200 combine_payload_bytes=277344256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277571456 ingress_bytes=387444736 link_bytes=387444736 token_bytes=554915712 weight_movement_bytes=110100480 total_nvlink_bytes=665016192 weight_to_token_ratio=0.198409 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.738301196e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278605824 dispatch_meta_bytes=229656 combine_payload_bytes=278605824 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=278835480 ingress_bytes=388706304 link_bytes=388706304 token_bytes=557441304 weight_movement_bytes=110100480 total_nvlink_bytes=667541784 weight_to_token_ratio=0.197510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760241939e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279007232 dispatch_meta_bytes=229560 combine_payload_bytes=279007232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279236792 ingress_bytes=389107712 link_bytes=389107712 token_bytes=558244024 weight_movement_bytes=110100480 total_nvlink_bytes=668344504 weight_to_token_ratio=0.197226 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767223084e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44390 payload_rows_remote=38888 dup_rows=21146 dup_rows_remote=18478 dedup_ratio=0.322662 router_tokens_per_expert=[519, 490, 526, 504, 503, 519, 513, 490, 513, 500, 522, 539, 488, 518, 503, 502, 515, 536, 517, 527, 513, 527, 529, 522, 468, 491, 549, 495, 520, 497, 536, 494, 553, 503, 518, 518, 505, 533, 491, 515, 510, 497, 493, 535, 549, 538, 475, 507, 530, 512, 546, 524, 545, 504, 519, 487, 478, 523, 499, 498, 524, 474, 499, 481] payload_rows_per_dst=[5518, 5502, 5690, 5490, 5594, 5478, 5660, 5458] dup_rows_per_dst=[2610, 2668, 2682, 2610, 2678, 2730, 2674, 2494] combine_rows_per_src=[5518, 5502, 5690, 5490, 5594, 5478, 5660, 5458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44604 payload_rows_remote=39014 dup_rows=20932 dup_rows_remote=18182 dedup_ratio=0.319397 router_tokens_per_expert=[493, 511, 502, 487, 470, 518, 522, 517, 539, 513, 492, 511, 526, 534, 498, 498, 524, 482, 510, 514, 517, 516, 499, 470, 522, 519, 463, 488, 518, 506, 532, 539, 489, 532, 512, 523, 509, 519, 508, 521, 494, 541, 502, 530, 527, 511, 491, 503, 513, 563, 488, 553, 503, 530, 481, 542, 505, 519, 544, 517, 505, 514, 523, 506] payload_rows_per_dst=[5566, 5572, 5566, 5562, 5588, 5562, 5598, 5590] dup_rows_per_dst=[2474, 2650, 2498, 2536, 2714, 2636, 2674, 2750] combine_rows_per_src=[5566, 5572, 5566, 5562, 5588, 5562, 5598, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=277544960 dispatch_meta_bytes=227768 combine_payload_bytes=277544960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277772728 ingress_bytes=387645440 link_bytes=387645440 token_bytes=555317688 weight_movement_bytes=110100480 total_nvlink_bytes=665418168 weight_to_token_ratio=0.198266 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.741791769e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38916 dup_rows=21088 dup_rows_remote=18582 dedup_ratio=0.321777 router_tokens_per_expert=[508, 485, 523, 519, 536, 516, 505, 492, 475, 516, 505, 512, 517, 463, 504, 518, 493, 534, 533, 508, 489, 504, 517, 540, 488, 512, 511, 532, 558, 537, 543, 513, 476, 557, 525, 489, 525, 489, 506, 521, 493, 486, 502, 513, 521, 484, 505, 515, 516, 532, 516, 492, 501, 508, 527, 531, 522, 511, 491, 494, 540, 542, 557, 475] payload_rows_per_dst=[5532, 5498, 5598, 5620, 5616, 5532, 5518, 5534] dup_rows_per_dst=[2636, 2522, 2638, 2768, 2560, 2506, 2728, 2730] combine_rows_per_src=[5532, 5498, 5598, 5620, 5616, 5532, 5518, 5534] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44582 payload_rows_remote=39060 dup_rows=20954 dup_rows_remote=18284 dedup_ratio=0.319733 router_tokens_per_expert=[497, 532, 511, 526, 557, 525, 510, 513, 569, 492, 513, 478, 515, 488, 495, 502, 520, 523, 539, 504, 519, 509, 526, 479, 538, 487, 499, 481, 521, 485, 521, 502, 513, 494, 506, 542, 511, 484, 514, 552, 519, 532, 465, 515, 484, 525, 510, 484, 499, 531, 508, 498, 518, 505, 500, 537, 526, 514, 501, 562, 511, 492, 514, 526] payload_rows_per_dst=[5672, 5482, 5552, 5520, 5638, 5504, 5522, 5692] dup_rows_per_dst=[2670, 2622, 2686, 2548, 2594, 2564, 2670, 2600] combine_rows_per_src=[5672, 5482, 5552, 5520, 5638, 5504, 5522, 5692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229464 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278978648 ingress_bytes=388849664 link_bytes=388849664 token_bytes=557727832 weight_movement_bytes=110100480 total_nvlink_bytes=667828312 weight_to_token_ratio=0.197409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762735205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38784 dup_rows=21160 dup_rows_remote=18556 dedup_ratio=0.322876 router_tokens_per_expert=[509, 487, 509, 544, 488, 527, 480, 505, 564, 496, 526, 513, 499, 534, 511, 539, 506, 479, 524, 513, 545, 534, 485, 533, 521, 519, 487, 487, 544, 506, 522, 512, 546, 484, 496, 495, 474, 523, 525, 508, 527, 496, 545, 510, 519, 462, 478, 556, 529, 517, 461, 517, 500, 516, 570, 546, 472, 459, 506, 545, 480, 545, 507, 506] payload_rows_per_dst=[5534, 5562, 5566, 5592, 5598, 5512, 5564, 5448] dup_rows_per_dst=[2564, 2802, 2672, 2604, 2504, 2674, 2748, 2592] combine_rows_per_src=[5534, 5562, 5566, 5592, 5598, 5512, 5564, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=228784 combine_payload_bytes=279652352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279881136 ingress_bytes=389752832 link_bytes=389752832 token_bytes=559533488 weight_movement_bytes=110100480 total_nvlink_bytes=669633968 weight_to_token_ratio=0.196772 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778442782e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=229992 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279179880 ingress_bytes=389050368 link_bytes=389050368 token_bytes=558129768 weight_movement_bytes=110100480 total_nvlink_bytes=668230248 weight_to_token_ratio=0.197267 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766225777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44414 payload_rows_remote=38842 dup_rows=21122 dup_rows_remote=18508 dedup_ratio=0.322296 router_tokens_per_expert=[510, 544, 534, 499, 534, 518, 521, 537, 538, 520, 525, 488, 464, 509, 519, 505, 501, 486, 524, 529, 492, 525, 516, 520, 523, 559, 478, 514, 536, 468, 486, 518, 517, 533, 500, 510, 478, 546, 484, 527, 516, 528, 469, 524, 540, 497, 534, 532, 481, 536, 453, 482, 486, 561, 530, 486, 499, 495, 536, 488, 537, 502, 502, 519] payload_rows_per_dst=[5582, 5478, 5572, 5546, 5500, 5630, 5560, 5546] dup_rows_per_dst=[2812, 2658, 2614, 2618, 2690, 2650, 2470, 2610] combine_rows_per_src=[5582, 5478, 5572, 5546, 5500, 5630, 5560, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279982080 dispatch_meta_bytes=229376 combine_payload_bytes=279982080 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=280211456 ingress_bytes=720384000 link_bytes=720384000 token_bytes=560193536 weight_movement_bytes=440401920 total_nvlink_bytes=1000595456 weight_to_token_ratio=0.786160 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252866259e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38916 dup_rows=21088 dup_rows_remote=18582 dedup_ratio=0.321777 router_tokens_per_expert=[508, 485, 523, 519, 536, 516, 505, 492, 475, 516, 505, 512, 517, 463, 504, 518, 493, 534, 533, 508, 489, 504, 517, 540, 488, 512, 511, 532, 558, 537, 543, 513, 476, 557, 525, 489, 525, 489, 506, 521, 493, 486, 502, 513, 521, 484, 505, 515, 516, 532, 516, 492, 501, 508, 527, 531, 522, 511, 491, 494, 540, 542, 557, 475] payload_rows_per_dst=[5532, 5498, 5598, 5620, 5616, 5532, 5518, 5534] dup_rows_per_dst=[2636, 2522, 2638, 2768, 2560, 2506, 2728, 2730] combine_rows_per_src=[5532, 5498, 5598, 5620, 5616, 5532, 5518, 5534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38770 dup_rows=21010 dup_rows_remote=18210 dedup_ratio=0.320587 router_tokens_per_expert=[569, 535, 471, 514, 495, 538, 487, 519, 506, 512, 523, 514, 520, 477, 512, 459, 522, 527, 518, 504, 523, 507, 484, 491, 511, 529, 535, 518, 514, 498, 531, 505, 517, 515, 503, 495, 537, 469, 496, 450, 524, 478, 488, 499, 532, 520, 534, 515, 524, 518, 554, 511, 527, 519, 569, 505, 513, 516, 513, 520, 547, 487, 498, 507] payload_rows_per_dst=[5756, 5584, 5466, 5586, 5402, 5680, 5412, 5640] dup_rows_per_dst=[2800, 2688, 2522, 2650, 2562, 2876, 2350, 2562] combine_rows_per_src=[5756, 5584, 5466, 5586, 5402, 5680, 5412, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44604 payload_rows_remote=39014 dup_rows=20932 dup_rows_remote=18182 dedup_ratio=0.319397 router_tokens_per_expert=[493, 511, 502, 487, 470, 518, 522, 517, 539, 513, 492, 511, 526, 534, 498, 498, 524, 482, 510, 514, 517, 516, 499, 470, 522, 519, 463, 488, 518, 506, 532, 539, 489, 532, 512, 523, 509, 519, 508, 521, 494, 541, 502, 530, 527, 511, 491, 503, 513, 563, 488, 553, 503, 530, 481, 542, 505, 519, 544, 517, 505, 514, 523, 506] payload_rows_per_dst=[5566, 5572, 5566, 5562, 5588, 5562, 5598, 5590] dup_rows_per_dst=[2474, 2650, 2498, 2536, 2714, 2636, 2674, 2750] combine_rows_per_src=[5566, 5572, 5566, 5562, 5588, 5562, 5598, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44376 payload_rows_remote=38784 dup_rows=21160 dup_rows_remote=18556 dedup_ratio=0.322876 router_tokens_per_expert=[509, 487, 509, 544, 488, 527, 480, 505, 564, 496, 526, 513, 499, 534, 511, 539, 506, 479, 524, 513, 545, 534, 485, 533, 521, 519, 487, 487, 544, 506, 522, 512, 546, 484, 496, 495, 474, 523, 525, 508, 527, 496, 545, 510, 519, 462, 478, 556, 529, 517, 461, 517, 500, 516, 570, 546, 472, 459, 506, 545, 480, 545, 507, 506] payload_rows_per_dst=[5534, 5562, 5566, 5592, 5598, 5512, 5564, 5448] dup_rows_per_dst=[2564, 2802, 2672, 2604, 2504, 2674, 2748, 2592] combine_rows_per_src=[5534, 5562, 5566, 5592, 5598, 5512, 5564, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44304 payload_rows_remote=38708 dup_rows=21232 dup_rows_remote=18530 dedup_ratio=0.323975 router_tokens_per_expert=[485, 482, 533, 485, 509, 471, 433, 507, 496, 539, 526, 526, 511, 477, 563, 486, 528, 499, 490, 520, 510, 546, 538, 476, 503, 524, 521, 552, 536, 520, 474, 513, 539, 522, 490, 511, 492, 559, 523, 513, 526, 494, 488, 489, 495, 485, 516, 508, 516, 518, 537, 539, 491, 540, 519, 534, 565, 514, 495, 500, 501, 523, 539, 508] payload_rows_per_dst=[5450, 5520, 5574, 5612, 5596, 5440, 5568, 5544] dup_rows_per_dst=[2360, 2728, 2640, 2674, 2702, 2562, 2820, 2746] combine_rows_per_src=[5450, 5520, 5574, 5612, 5596, 5440, 5568, 5544] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38928 dup_rows=21100 dup_rows_remote=18430 dedup_ratio=0.321960 router_tokens_per_expert=[520, 507, 518, 512, 520, 552, 503, 509, 509, 517, 512, 533, 498, 506, 534, 502, 533, 493, 526, 515, 500, 509, 525, 514, 496, 499, 514, 498, 506, 520, 519, 526, 500, 521, 515, 492, 518, 504, 512, 527, 482, 493, 520, 511, 545, 472, 547, 532, 475, 480, 480, 540, 509, 544, 526, 530, 527, 509, 502, 480, 526, 509, 518, 477] payload_rows_per_dst=[5618, 5580, 5532, 5598, 5508, 5508, 5580, 5512] dup_rows_per_dst=[2664, 2642, 2698, 2558, 2670, 2696, 2588, 2584] combine_rows_per_src=[5618, 5580, 5532, 5598, 5508, 5508, 5580, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44526 payload_rows_remote=38770 dup_rows=21010 dup_rows_remote=18210 dedup_ratio=0.320587 router_tokens_per_expert=[569, 535, 471, 514, 495, 538, 487, 519, 506, 512, 523, 514, 520, 477, 512, 459, 522, 527, 518, 504, 523, 507, 484, 491, 511, 529, 535, 518, 514, 498, 531, 505, 517, 515, 503, 495, 537, 469, 496, 450, 524, 478, 488, 499, 532, 520, 534, 515, 524, 518, 554, 511, 527, 519, 569, 505, 513, 516, 513, 520, 547, 487, 498, 507] payload_rows_per_dst=[5756, 5584, 5466, 5586, 5402, 5680, 5412, 5640] dup_rows_per_dst=[2800, 2688, 2522, 2650, 2562, 2876, 2350, 2562] combine_rows_per_src=[5756, 5584, 5466, 5586, 5402, 5680, 5412, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=229400 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278648856 ingress_bytes=498620416 link_bytes=498620416 token_bytes=557068312 weight_movement_bytes=220200960 total_nvlink_bytes=777269272 weight_to_token_ratio=0.395285 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671829124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279652352 dispatch_meta_bytes=228784 combine_payload_bytes=279652352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279881136 ingress_bytes=389752832 link_bytes=389752832 token_bytes=559533488 weight_movement_bytes=110100480 total_nvlink_bytes=669633968 weight_to_token_ratio=0.196772 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.778442782e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278003712 dispatch_meta_bytes=229360 combine_payload_bytes=278003712 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=278233072 ingress_bytes=608305152 link_bytes=608305152 token_bytes=556236784 weight_movement_bytes=330301440 total_nvlink_bytes=886538224 weight_to_token_ratio=0.593814 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057942708e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278419456 dispatch_meta_bytes=229400 combine_payload_bytes=278419456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278648856 ingress_bytes=498620416 link_bytes=498620416 token_bytes=557068312 weight_movement_bytes=220200960 total_nvlink_bytes=777269272 weight_to_token_ratio=0.395285 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.671829124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278949888 dispatch_meta_bytes=229992 combine_payload_bytes=278949888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279179880 ingress_bytes=389050368 link_bytes=389050368 token_bytes=558129768 weight_movement_bytes=110100480 total_nvlink_bytes=668230248 weight_to_token_ratio=0.197267 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766225777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277903360 dispatch_meta_bytes=227920 combine_payload_bytes=277903360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278131280 ingress_bytes=388003840 link_bytes=388003840 token_bytes=556034640 weight_movement_bytes=110100480 total_nvlink_bytes=666135120 weight_to_token_ratio=0.198010 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748024934e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279035904 dispatch_meta_bytes=229432 combine_payload_bytes=279035904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279265336 ingress_bytes=389136384 link_bytes=389136384 token_bytes=558301240 weight_movement_bytes=110100480 total_nvlink_bytes=668401720 weight_to_token_ratio=0.197206 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767721737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44464 payload_rows_remote=38864 dup_rows=21072 dup_rows_remote=18500 dedup_ratio=0.321533 router_tokens_per_expert=[515, 525, 511, 535, 518, 487, 526, 511, 558, 513, 507, 512, 536, 466, 501, 493, 510, 487, 521, 510, 535, 527, 556, 508, 496, 531, 532, 514, 538, 538, 519, 541, 453, 517, 510, 503, 494, 494, 469, 521, 499, 500, 522, 483, 493, 496, 542, 475, 518, 519, 525, 549, 529, 525, 515, 478, 506, 514, 516, 472, 513, 504, 471, 566] payload_rows_per_dst=[5608, 5600, 5594, 5596, 5494, 5478, 5588, 5506] dup_rows_per_dst=[2648, 2572, 2714, 2822, 2428, 2542, 2728, 2618] combine_rows_per_src=[5608, 5600, 5594, 5596, 5494, 5478, 5588, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44324 payload_rows_remote=38764 dup_rows=21212 dup_rows_remote=18526 dedup_ratio=0.323669 router_tokens_per_expert=[537, 473, 536, 505, 513, 512, 483, 500, 526, 522, 535, 545, 505, 513, 495, 538, 518, 505, 509, 504, 521, 523, 481, 553, 494, 508, 518, 482, 462, 487, 500, 506, 496, 531, 518, 523, 516, 530, 528, 487, 513, 506, 513, 494, 527, 491, 533, 536, 508, 523, 521, 491, 507, 539, 520, 541, 472, 542, 480, 551, 490, 524, 501, 507] payload_rows_per_dst=[5404, 5362, 5246, 5642, 5856, 5632, 5622, 5560] dup_rows_per_dst=[2470, 2402, 2382, 2776, 3056, 2714, 2726, 2686] combine_rows_per_src=[5404, 5362, 5246, 5642, 5856, 5632, 5622, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277458944 dispatch_meta_bytes=228952 combine_payload_bytes=277458944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277687896 ingress_bytes=387559424 link_bytes=387559424 token_bytes=555146840 weight_movement_bytes=110100480 total_nvlink_bytes=665247320 weight_to_token_ratio=0.198327 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.740295809e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44614 payload_rows_remote=39034 dup_rows=20922 dup_rows_remote=18446 dedup_ratio=0.319244 router_tokens_per_expert=[479, 524, 541, 536, 489, 525, 541, 521, 538, 489, 507, 556, 518, 523, 500, 510, 478, 497, 513, 535, 538, 540, 527, 499, 491, 468, 518, 537, 493, 496, 529, 490, 478, 508, 493, 510, 490, 523, 512, 521, 508, 518, 534, 466, 468, 526, 501, 507, 483, 519, 545, 477, 479, 516, 507, 538, 484, 537, 528, 561, 490, 536, 550, 509] payload_rows_per_dst=[5654, 5644, 5556, 5492, 5540, 5580, 5530, 5618] dup_rows_per_dst=[2658, 2638, 2698, 2552, 2530, 2476, 2598, 2772] combine_rows_per_src=[5654, 5644, 5556, 5492, 5540, 5580, 5530, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277903360 dispatch_meta_bytes=227920 combine_payload_bytes=277903360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278131280 ingress_bytes=388003840 link_bytes=388003840 token_bytes=556034640 weight_movement_bytes=110100480 total_nvlink_bytes=666135120 weight_to_token_ratio=0.198010 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.748024934e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44426 payload_rows_remote=38888 dup_rows=21110 dup_rows_remote=18506 dedup_ratio=0.322113 router_tokens_per_expert=[518, 510, 530, 498, 512, 530, 521, 504, 504, 507, 520, 531, 537, 529, 515, 489, 529, 528, 515, 508, 491, 502, 514, 498, 497, 483, 518, 536, 494, 509, 470, 500, 478, 498, 509, 489, 518, 530, 510, 528, 561, 515, 528, 522, 497, 502, 496, 507, 506, 472, 511, 536, 499, 516, 501, 530, 539, 566, 510, 514, 507, 501, 519, 506] payload_rows_per_dst=[5570, 5528, 5570, 5530, 5472, 5640, 5538, 5578] dup_rows_per_dst=[2676, 2736, 2600, 2484, 2648, 2616, 2604, 2746] combine_rows_per_src=[5570, 5528, 5570, 5530, 5472, 5640, 5538, 5578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229456 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278806608 ingress_bytes=718979072 link_bytes=718979072 token_bytes=557383760 weight_movement_bytes=440401920 total_nvlink_bytes=997785680 weight_to_token_ratio=0.790123 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250422858e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44438 payload_rows_remote=38886 dup_rows=21098 dup_rows_remote=18468 dedup_ratio=0.321930 router_tokens_per_expert=[490, 500, 518, 508, 503, 515, 499, 529, 540, 537, 500, 541, 541, 529, 536, 518, 513, 495, 521, 514, 513, 581, 497, 518, 532, 501, 486, 495, 502, 537, 513, 525, 479, 480, 499, 517, 532, 509, 498, 485, 495, 514, 549, 463, 472, 535, 503, 487, 490, 536, 535, 479, 510, 524, 481, 516, 485, 504, 520, 526, 526, 487, 559, 526] payload_rows_per_dst=[5548, 5736, 5554, 5552, 5480, 5450, 5570, 5548] dup_rows_per_dst=[2576, 2748, 2750, 2630, 2518, 2586, 2572, 2718] combine_rows_per_src=[5548, 5736, 5554, 5552, 5480, 5450, 5570, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277860352 dispatch_meta_bytes=229160 combine_payload_bytes=277860352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278089512 ingress_bytes=387960832 link_bytes=387960832 token_bytes=555949864 weight_movement_bytes=110100480 total_nvlink_bytes=666050344 weight_to_token_ratio=0.198040 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747276954e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278577152 dispatch_meta_bytes=229456 combine_payload_bytes=278577152 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=278806608 ingress_bytes=718979072 link_bytes=718979072 token_bytes=557383760 weight_movement_bytes=440401920 total_nvlink_bytes=997785680 weight_to_token_ratio=0.790123 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.250422858e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229920 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280025632 ingress_bytes=389896192 link_bytes=389896192 token_bytes=559821344 weight_movement_bytes=110100480 total_nvlink_bytes=669921824 weight_to_token_ratio=0.196671 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780936048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229576 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278978760 ingress_bytes=388849664 link_bytes=388849664 token_bytes=557727944 weight_movement_bytes=110100480 total_nvlink_bytes=667828424 weight_to_token_ratio=0.197409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762735205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44490 payload_rows_remote=38808 dup_rows=21046 dup_rows_remote=18322 dedup_ratio=0.321136 router_tokens_per_expert=[532, 497, 543, 512, 460, 511, 536, 510, 523, 536, 486, 514, 501, 515, 514, 515, 546, 533, 550, 532, 495, 517, 515, 515, 530, 515, 485, 509, 490, 531, 500, 503, 476, 507, 527, 513, 513, 558, 510, 508, 538, 504, 518, 536, 501, 501, 494, 512, 491, 522, 545, 479, 488, 518, 509, 518, 485, 514, 501, 494, 491, 529, 467, 530] payload_rows_per_dst=[5542, 5556, 5682, 5576, 5608, 5482, 5558, 5486] dup_rows_per_dst=[2660, 2652, 2724, 2550, 2616, 2726, 2582, 2536] combine_rows_per_src=[5542, 5556, 5682, 5576, 5608, 5482, 5558, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44324 payload_rows_remote=38764 dup_rows=21212 dup_rows_remote=18526 dedup_ratio=0.323669 router_tokens_per_expert=[537, 473, 536, 505, 513, 512, 483, 500, 526, 522, 535, 545, 505, 513, 495, 538, 518, 505, 509, 504, 521, 523, 481, 553, 494, 508, 518, 482, 462, 487, 500, 506, 496, 531, 518, 523, 516, 530, 528, 487, 513, 506, 513, 494, 527, 491, 533, 536, 508, 523, 521, 491, 507, 539, 520, 541, 472, 542, 480, 551, 490, 524, 501, 507] payload_rows_per_dst=[5404, 5362, 5246, 5642, 5856, 5632, 5622, 5560] dup_rows_per_dst=[2470, 2402, 2382, 2776, 3056, 2714, 2726, 2686] combine_rows_per_src=[5404, 5362, 5246, 5642, 5856, 5632, 5622, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44438 payload_rows_remote=38886 dup_rows=21098 dup_rows_remote=18468 dedup_ratio=0.321930 router_tokens_per_expert=[490, 500, 518, 508, 503, 515, 499, 529, 540, 537, 500, 541, 541, 529, 536, 518, 513, 495, 521, 514, 513, 581, 497, 518, 532, 501, 486, 495, 502, 537, 513, 525, 479, 480, 499, 517, 532, 509, 498, 485, 495, 514, 549, 463, 472, 535, 503, 487, 490, 536, 535, 479, 510, 524, 481, 516, 485, 504, 520, 526, 526, 487, 559, 526] payload_rows_per_dst=[5548, 5736, 5554, 5552, 5480, 5450, 5570, 5548] dup_rows_per_dst=[2576, 2748, 2750, 2630, 2518, 2586, 2572, 2718] combine_rows_per_src=[5548, 5736, 5554, 5552, 5480, 5450, 5570, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44436 payload_rows_remote=38928 dup_rows=21100 dup_rows_remote=18430 dedup_ratio=0.321960 router_tokens_per_expert=[520, 507, 518, 512, 520, 552, 503, 509, 509, 517, 512, 533, 498, 506, 534, 502, 533, 493, 526, 515, 500, 509, 525, 514, 496, 499, 514, 498, 506, 520, 519, 526, 500, 521, 515, 492, 518, 504, 512, 527, 482, 493, 520, 511, 545, 472, 547, 532, 475, 480, 480, 540, 509, 544, 526, 530, 527, 509, 502, 480, 526, 509, 518, 477] payload_rows_per_dst=[5618, 5580, 5532, 5598, 5508, 5508, 5580, 5512] dup_rows_per_dst=[2664, 2642, 2698, 2558, 2670, 2696, 2588, 2584] combine_rows_per_src=[5618, 5580, 5532, 5598, 5508, 5508, 5580, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278749184 dispatch_meta_bytes=229576 combine_payload_bytes=278749184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278978760 ingress_bytes=388849664 link_bytes=388849664 token_bytes=557727944 weight_movement_bytes=110100480 total_nvlink_bytes=667828424 weight_to_token_ratio=0.197409 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762735205e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44626 payload_rows_remote=39068 dup_rows=20910 dup_rows_remote=18468 dedup_ratio=0.319061 router_tokens_per_expert=[515, 521, 509, 516, 514, 514, 506, 522, 502, 500, 527, 512, 530, 517, 501, 479, 500, 529, 486, 487, 482, 486, 475, 509, 536, 520, 546, 527, 472, 537, 497, 507, 491, 480, 519, 504, 524, 489, 490, 503, 539, 497, 489, 523, 531, 511, 526, 521, 513, 562, 515, 532, 538, 539, 523, 507, 543, 512, 488, 473, 522, 519, 531, 533] payload_rows_per_dst=[5580, 5568, 5518, 5550, 5558, 5632, 5674, 5546] dup_rows_per_dst=[2654, 2568, 2390, 2734, 2442, 2642, 2784, 2696] combine_rows_per_src=[5580, 5568, 5518, 5550, 5558, 5632, 5674, 5546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44614 payload_rows_remote=39034 dup_rows=20922 dup_rows_remote=18446 dedup_ratio=0.319244 router_tokens_per_expert=[479, 524, 541, 536, 489, 525, 541, 521, 538, 489, 507, 556, 518, 523, 500, 510, 478, 497, 513, 535, 538, 540, 527, 499, 491, 468, 518, 537, 493, 496, 529, 490, 478, 508, 493, 510, 490, 523, 512, 521, 508, 518, 534, 466, 468, 526, 501, 507, 483, 519, 545, 477, 479, 516, 507, 538, 484, 537, 528, 561, 490, 536, 550, 509] payload_rows_per_dst=[5654, 5644, 5556, 5492, 5540, 5580, 5530, 5618] dup_rows_per_dst=[2658, 2638, 2698, 2552, 2530, 2476, 2598, 2772] combine_rows_per_src=[5654, 5644, 5556, 5492, 5540, 5580, 5530, 5618] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44626 payload_rows_remote=39042 dup_rows=20910 dup_rows_remote=18254 dedup_ratio=0.319061 router_tokens_per_expert=[521, 454, 529, 509, 503, 525, 556, 523, 492, 521, 561, 486, 496, 508, 517, 489, 499, 547, 527, 521, 514, 522, 473, 515, 521, 527, 471, 513, 543, 531, 489, 494, 510, 485, 489, 545, 503, 525, 497, 502, 501, 513, 527, 562, 492, 574, 525, 511, 510, 495, 518, 503, 529, 489, 511, 521, 505, 506, 536, 528, 465, 501, 516, 477] payload_rows_per_dst=[5584, 5530, 5572, 5600, 5508, 5740, 5568, 5524] dup_rows_per_dst=[2656, 2610, 2664, 2578, 2604, 2670, 2584, 2544] combine_rows_per_src=[5584, 5530, 5572, 5600, 5508, 5740, 5568, 5524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278175744 dispatch_meta_bytes=228520 combine_payload_bytes=278175744 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278404264 ingress_bytes=498376704 link_bytes=498376704 token_bytes=556580008 weight_movement_bytes=220200960 total_nvlink_bytes=776780968 weight_to_token_ratio=0.395632 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.667590571e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277860352 dispatch_meta_bytes=229160 combine_payload_bytes=277860352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278089512 ingress_bytes=387960832 link_bytes=387960832 token_bytes=555949864 weight_movement_bytes=110100480 total_nvlink_bytes=666050344 weight_to_token_ratio=0.198040 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747276954e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278734848 dispatch_meta_bytes=229416 combine_payload_bytes=278734848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278964264 ingress_bytes=388835328 link_bytes=388835328 token_bytes=557699112 weight_movement_bytes=110100480 total_nvlink_bytes=667799592 weight_to_token_ratio=0.197419 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.762485878e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278175744 dispatch_meta_bytes=228520 combine_payload_bytes=278175744 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=278404264 ingress_bytes=498376704 link_bytes=498376704 token_bytes=556580008 weight_movement_bytes=220200960 total_nvlink_bytes=776780968 weight_to_token_ratio=0.395632 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.667590571e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280039424 dispatch_meta_bytes=230144 combine_payload_bytes=280039424 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280269568 ingress_bytes=500240384 link_bytes=500240384 token_bytes=560308992 weight_movement_bytes=220200960 total_nvlink_bytes=780509952 weight_to_token_ratio=0.392999 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.700003032e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44320 payload_rows_remote=38786 dup_rows=21216 dup_rows_remote=18610 dedup_ratio=0.323730 router_tokens_per_expert=[515, 493, 510, 498, 507, 540, 477, 519, 514, 497, 509, 508, 490, 526, 524, 502, 505, 503, 525, 499, 513, 525, 515, 523, 513, 495, 523, 494, 541, 536, 495, 526, 509, 476, 521, 538, 529, 503, 482, 547, 565, 490, 519, 554, 514, 506, 514, 511, 480, 512, 507, 526, 486, 516, 514, 532, 556, 483, 516, 469, 520, 475, 518, 520] payload_rows_per_dst=[5512, 5534, 5580, 5548, 5498, 5582, 5572, 5494] dup_rows_per_dst=[2606, 2606, 2636, 2698, 2712, 2764, 2574, 2620] combine_rows_per_src=[5512, 5534, 5580, 5548, 5498, 5582, 5572, 5494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279795712 dispatch_meta_bytes=229920 combine_payload_bytes=279795712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280025632 ingress_bytes=389896192 link_bytes=389896192 token_bytes=559821344 weight_movement_bytes=110100480 total_nvlink_bytes=669921824 weight_to_token_ratio=0.196671 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.780936048e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279035904 dispatch_meta_bytes=229432 combine_payload_bytes=279035904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279265336 ingress_bytes=389136384 link_bytes=389136384 token_bytes=558301240 weight_movement_bytes=110100480 total_nvlink_bytes=668401720 weight_to_token_ratio=0.197206 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.767721737e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=38940 dup_rows=20946 dup_rows_remote=18254 dedup_ratio=0.319611 router_tokens_per_expert=[517, 522, 545, 535, 518, 487, 522, 524, 507, 538, 477, 498, 463, 508, 539, 484, 514, 561, 545, 537, 499, 523, 506, 480, 511, 477, 481, 494, 514, 466, 555, 501, 533, 499, 517, 508, 489, 500, 484, 485, 563, 513, 508, 493, 498, 488, 558, 550, 494, 490, 523, 531, 530, 525, 566, 508, 492, 502, 479, 501, 513, 519, 509, 552] payload_rows_per_dst=[5648, 5446, 5694, 5478, 5490, 5650, 5618, 5566] dup_rows_per_dst=[2692, 2582, 2636, 2520, 2540, 2692, 2716, 2568] combine_rows_per_src=[5648, 5446, 5694, 5478, 5490, 5650, 5618, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=39092 dup_rows=20936 dup_rows_remote=18430 dedup_ratio=0.319458 router_tokens_per_expert=[485, 507, 506, 524, 507, 515, 506, 503, 498, 577, 495, 491, 502, 527, 551, 532, 498, 520, 525, 505, 519, 486, 537, 511, 525, 529, 502, 506, 493, 525, 491, 505, 519, 509, 524, 461, 564, 530, 492, 474, 506, 524, 518, 530, 525, 526, 525, 535, 494, 492, 494, 523, 513, 512, 473, 506, 540, 505, 504, 504, 514, 502, 503, 524] payload_rows_per_dst=[5548, 5620, 5588, 5564, 5540, 5654, 5508, 5578] dup_rows_per_dst=[2558, 2726, 2614, 2588, 2606, 2724, 2506, 2614] combine_rows_per_src=[5548, 5620, 5588, 5564, 5540, 5654, 5508, 5578] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279853056 dispatch_meta_bytes=229184 combine_payload_bytes=279853056 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=280082240 ingress_bytes=610154496 link_bytes=610154496 token_bytes=559935296 weight_movement_bytes=330301440 total_nvlink_bytes=890236736 weight_to_token_ratio=0.589892 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061159022e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=38852 dup_rows=20938 dup_rows_remote=18254 dedup_ratio=0.319489 router_tokens_per_expert=[500, 542, 502, 518, 503, 539, 507, 529, 527, 517, 527, 534, 497, 532, 532, 510, 518, 465, 494, 509, 497, 471, 515, 509, 517, 481, 527, 533, 506, 545, 525, 510, 523, 524, 529, 535, 511, 497, 515, 520, 507, 495, 514, 511, 531, 528, 500, 496, 516, 508, 547, 487, 501, 486, 510, 484, 509, 516, 502, 516, 527, 508, 490, 487] payload_rows_per_dst=[5526, 5660, 5540, 5576, 5608, 5534, 5408, 5746] dup_rows_per_dst=[2554, 2872, 2416, 2712, 2714, 2598, 2388, 2684] combine_rows_per_src=[5526, 5660, 5540, 5576, 5608, 5534, 5408, 5746] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44600 payload_rows_remote=39092 dup_rows=20936 dup_rows_remote=18430 dedup_ratio=0.319458 router_tokens_per_expert=[485, 507, 506, 524, 507, 515, 506, 503, 498, 577, 495, 491, 502, 527, 551, 532, 498, 520, 525, 505, 519, 486, 537, 511, 525, 529, 502, 506, 493, 525, 491, 505, 519, 509, 524, 461, 564, 530, 492, 474, 506, 524, 518, 530, 525, 526, 525, 535, 494, 492, 494, 523, 513, 512, 473, 506, 540, 505, 504, 504, 514, 502, 503, 524] payload_rows_per_dst=[5548, 5620, 5588, 5564, 5540, 5654, 5508, 5578] dup_rows_per_dst=[2558, 2726, 2614, 2588, 2606, 2724, 2506, 2614] combine_rows_per_src=[5548, 5620, 5588, 5564, 5540, 5654, 5508, 5578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278018048 dispatch_meta_bytes=229584 combine_payload_bytes=278018048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278247632 ingress_bytes=388118528 link_bytes=388118528 token_bytes=556265680 weight_movement_bytes=110100480 total_nvlink_bytes=666366160 weight_to_token_ratio=0.197928 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750019547e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=39002 dup_rows=21036 dup_rows_remote=18506 dedup_ratio=0.320984 router_tokens_per_expert=[510, 490, 502, 528, 520, 481, 541, 499, 517, 522, 537, 497, 513, 521, 514, 508, 495, 491, 510, 491, 508, 497, 496, 491, 500, 446, 480, 514, 527, 545, 490, 512, 568, 506, 547, 537, 460, 535, 518, 529, 511, 498, 505, 529, 501, 523, 563, 521, 516, 511, 529, 509, 520, 487, 529, 493, 545, 527, 523, 501, 513, 486, 519, 516] payload_rows_per_dst=[5508, 5548, 5518, 5498, 5660, 5624, 5536, 5608] dup_rows_per_dst=[2634, 2710, 2440, 2530, 2740, 2678, 2652, 2652] combine_rows_per_src=[5508, 5548, 5518, 5498, 5660, 5624, 5536, 5608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=228424 combine_payload_bytes=278491136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278719560 ingress_bytes=388591616 link_bytes=388591616 token_bytes=557210696 weight_movement_bytes=110100480 total_nvlink_bytes=667311176 weight_to_token_ratio=0.197592 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758247326e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278018048 dispatch_meta_bytes=229584 combine_payload_bytes=278018048 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278247632 ingress_bytes=388118528 link_bytes=388118528 token_bytes=556265680 weight_movement_bytes=110100480 total_nvlink_bytes=666366160 weight_to_token_ratio=0.197928 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750019547e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=228776 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279350696 ingress_bytes=609423360 link_bytes=609423360 token_bytes=558472616 weight_movement_bytes=330301440 total_nvlink_bytes=888774056 weight_to_token_ratio=0.591437 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059887456e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280211456 dispatch_meta_bytes=230088 combine_payload_bytes=280211456 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280441544 ingress_bytes=500412416 link_bytes=500412416 token_bytes=560653000 weight_movement_bytes=220200960 total_nvlink_bytes=780853960 weight_to_token_ratio=0.392758 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.702994951e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38846 dup_rows=21108 dup_rows_remote=18364 dedup_ratio=0.322083 router_tokens_per_expert=[513, 530, 472, 504, 549, 545, 521, 488, 551, 540, 529, 531, 448, 528, 510, 575, 497, 493, 507, 501, 483, 532, 497, 481, 542, 532, 508, 503, 526, 487, 506, 499, 513, 514, 532, 530, 521, 508, 524, 521, 498, 508, 520, 483, 513, 495, 516, 503, 500, 497, 535, 526, 523, 527, 485, 509, 514, 494, 511, 476, 529, 513, 527, 475] payload_rows_per_dst=[5582, 5690, 5482, 5580, 5582, 5444, 5592, 5476] dup_rows_per_dst=[2662, 2734, 2500, 2626, 2744, 2628, 2612, 2602] combine_rows_per_src=[5582, 5690, 5482, 5580, 5582, 5444, 5592, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44598 payload_rows_remote=38852 dup_rows=20938 dup_rows_remote=18254 dedup_ratio=0.319489 router_tokens_per_expert=[500, 542, 502, 518, 503, 539, 507, 529, 527, 517, 527, 534, 497, 532, 532, 510, 518, 465, 494, 509, 497, 471, 515, 509, 517, 481, 527, 533, 506, 545, 525, 510, 523, 524, 529, 535, 511, 497, 515, 520, 507, 495, 514, 511, 531, 528, 500, 496, 516, 508, 547, 487, 501, 486, 510, 484, 509, 516, 502, 516, 527, 508, 490, 487] payload_rows_per_dst=[5526, 5660, 5540, 5576, 5608, 5534, 5408, 5746] dup_rows_per_dst=[2554, 2872, 2416, 2712, 2714, 2598, 2388, 2684] combine_rows_per_src=[5526, 5660, 5540, 5576, 5608, 5534, 5408, 5746] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44430 payload_rows_remote=38874 dup_rows=21106 dup_rows_remote=18560 dedup_ratio=0.322052 router_tokens_per_expert=[554, 537, 503, 552, 510, 501, 523, 518, 504, 527, 480, 500, 479, 506, 518, 513, 526, 440, 535, 492, 489, 518, 529, 522, 521, 514, 510, 525, 530, 497, 526, 524, 490, 514, 513, 554, 494, 500, 500, 519, 496, 519, 466, 510, 518, 558, 507, 531, 480, 541, 533, 558, 475, 524, 520, 521, 513, 521, 485, 493, 503, 496, 486, 507] payload_rows_per_dst=[5766, 5456, 5556, 5544, 5504, 5578, 5520, 5506] dup_rows_per_dst=[2630, 2598, 2546, 2750, 2664, 2632, 2784, 2502] combine_rows_per_src=[5766, 5456, 5556, 5544, 5504, 5578, 5520, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44590 payload_rows_remote=38940 dup_rows=20946 dup_rows_remote=18254 dedup_ratio=0.319611 router_tokens_per_expert=[517, 522, 545, 535, 518, 487, 522, 524, 507, 538, 477, 498, 463, 508, 539, 484, 514, 561, 545, 537, 499, 523, 506, 480, 511, 477, 481, 494, 514, 466, 555, 501, 533, 499, 517, 508, 489, 500, 484, 485, 563, 513, 508, 493, 498, 488, 558, 550, 494, 490, 523, 531, 530, 525, 566, 508, 492, 502, 479, 501, 513, 519, 509, 552] payload_rows_per_dst=[5648, 5446, 5694, 5478, 5490, 5650, 5618, 5566] dup_rows_per_dst=[2692, 2582, 2636, 2520, 2540, 2692, 2716, 2568] combine_rows_per_src=[5648, 5446, 5694, 5478, 5490, 5650, 5618, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44500 payload_rows_remote=39002 dup_rows=21036 dup_rows_remote=18506 dedup_ratio=0.320984 router_tokens_per_expert=[510, 490, 502, 528, 520, 481, 541, 499, 517, 522, 537, 497, 513, 521, 514, 508, 495, 491, 510, 491, 508, 497, 496, 491, 500, 446, 480, 514, 527, 545, 490, 512, 568, 506, 547, 537, 460, 535, 518, 529, 511, 498, 505, 529, 501, 523, 563, 521, 516, 511, 529, 509, 520, 487, 529, 493, 545, 527, 523, 501, 513, 486, 519, 516] payload_rows_per_dst=[5508, 5548, 5518, 5498, 5660, 5624, 5536, 5608] dup_rows_per_dst=[2634, 2710, 2440, 2530, 2740, 2678, 2652, 2652] combine_rows_per_src=[5508, 5548, 5518, 5498, 5660, 5624, 5536, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44626 payload_rows_remote=39068 dup_rows=20910 dup_rows_remote=18468 dedup_ratio=0.319061 router_tokens_per_expert=[515, 521, 509, 516, 514, 514, 506, 522, 502, 500, 527, 512, 530, 517, 501, 479, 500, 529, 486, 487, 482, 486, 475, 509, 536, 520, 546, 527, 472, 537, 497, 507, 491, 480, 519, 504, 524, 489, 490, 503, 539, 497, 489, 523, 531, 511, 526, 521, 513, 562, 515, 532, 538, 539, 523, 507, 543, 512, 488, 473, 522, 519, 531, 533] payload_rows_per_dst=[5580, 5568, 5518, 5550, 5558, 5632, 5674, 5546] dup_rows_per_dst=[2654, 2568, 2390, 2734, 2442, 2642, 2784, 2696] combine_rows_per_src=[5580, 5568, 5518, 5550, 5558, 5632, 5674, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44572 payload_rows_remote=39010 dup_rows=20964 dup_rows_remote=18406 dedup_ratio=0.319885 router_tokens_per_expert=[528, 508, 514, 507, 513, 499, 509, 482, 526, 509, 500, 493, 512, 530, 495, 456, 494, 493, 532, 525, 510, 482, 485, 550, 471, 509, 496, 525, 499, 530, 500, 521, 511, 533, 484, 548, 573, 502, 551, 500, 540, 529, 509, 537, 531, 483, 462, 546, 510, 509, 495, 546, 499, 553, 525, 511, 530, 496, 527, 496, 480, 527, 517, 505] payload_rows_per_dst=[5562, 5514, 5792, 5630, 5570, 5372, 5578, 5554] dup_rows_per_dst=[2558, 2528, 3072, 2616, 2690, 2180, 2718, 2602] combine_rows_per_src=[5562, 5514, 5792, 5630, 5570, 5372, 5578, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279566336 dispatch_meta_bytes=230032 combine_payload_bytes=279566336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279796368 ingress_bytes=389666816 link_bytes=389666816 token_bytes=559362704 weight_movement_bytes=110100480 total_nvlink_bytes=669463184 weight_to_token_ratio=0.196832 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776946822e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278491136 dispatch_meta_bytes=228424 combine_payload_bytes=278491136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278719560 ingress_bytes=388591616 link_bytes=388591616 token_bytes=557210696 weight_movement_bytes=110100480 total_nvlink_bytes=667311176 weight_to_token_ratio=0.197592 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.758247326e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44646 payload_rows_remote=39078 dup_rows=20890 dup_rows_remote=18308 dedup_ratio=0.318756 router_tokens_per_expert=[543, 538, 529, 532, 508, 522, 520, 524, 501, 535, 496, 482, 513, 492, 489, 567, 524, 514, 519, 520, 498, 487, 522, 477, 557, 509, 511, 497, 501, 526, 532, 501, 492, 510, 510, 501, 501, 534, 518, 501, 455, 530, 527, 523, 502, 492, 493, 496, 532, 490, 540, 534, 551, 520, 506, 490, 449, 487, 535, 524, 504, 535, 520, 480] payload_rows_per_dst=[5720, 5568, 5558, 5592, 5554, 5538, 5554, 5562] dup_rows_per_dst=[2712, 2582, 2564, 2676, 2580, 2618, 2652, 2506] combine_rows_per_src=[5720, 5568, 5558, 5592, 5554, 5538, 5554, 5562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278448128 dispatch_meta_bytes=228840 combine_payload_bytes=278448128 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278676968 ingress_bytes=498649088 link_bytes=498649088 token_bytes=557125096 weight_movement_bytes=220200960 total_nvlink_bytes=777326056 weight_to_token_ratio=0.395245 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672327777e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278648832 dispatch_meta_bytes=229736 combine_payload_bytes=278648832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278878568 ingress_bytes=388749312 link_bytes=388749312 token_bytes=557527400 weight_movement_bytes=110100480 total_nvlink_bytes=667627880 weight_to_token_ratio=0.197480 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.760989918e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279121920 dispatch_meta_bytes=228776 combine_payload_bytes=279121920 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=279350696 ingress_bytes=609423360 link_bytes=609423360 token_bytes=558472616 weight_movement_bytes=330301440 total_nvlink_bytes=888774056 weight_to_token_ratio=0.591437 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059887456e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=229664 combine_payload_bytes=279623680 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279853344 ingress_bytes=609925120 link_bytes=609925120 token_bytes=559477024 weight_movement_bytes=330301440 total_nvlink_bytes=889778464 weight_to_token_ratio=0.590375 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.051471 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060760099e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38790 dup_rows=21088 dup_rows_remote=18144 dedup_ratio=0.321777 router_tokens_per_expert=[506, 483, 484, 496, 481, 524, 509, 523, 486, 516, 484, 516, 528, 557, 502, 517, 525, 508, 528, 503, 525, 545, 481, 518, 523, 505, 489, 499, 499, 557, 514, 508, 521, 527, 541, 475, 492, 462, 508, 502, 519, 518, 511, 496, 524, 498, 483, 517, 552, 529, 543, 500, 492, 532, 504, 541, 525, 511, 513, 496, 546, 485, 528, 538] payload_rows_per_dst=[5244, 5444, 5618, 5646, 5540, 5474, 5824, 5658] dup_rows_per_dst=[2124, 2450, 2648, 2728, 2516, 2658, 3020, 2944] combine_rows_per_src=[5244, 5444, 5618, 5646, 5540, 5474, 5824, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44646 payload_rows_remote=39078 dup_rows=20890 dup_rows_remote=18308 dedup_ratio=0.318756 router_tokens_per_expert=[543, 538, 529, 532, 508, 522, 520, 524, 501, 535, 496, 482, 513, 492, 489, 567, 524, 514, 519, 520, 498, 487, 522, 477, 557, 509, 511, 497, 501, 526, 532, 501, 492, 510, 510, 501, 501, 534, 518, 501, 455, 530, 527, 523, 502, 492, 493, 496, 532, 490, 540, 534, 551, 520, 506, 490, 449, 487, 535, 524, 504, 535, 520, 480] payload_rows_per_dst=[5720, 5568, 5558, 5592, 5554, 5538, 5554, 5562] dup_rows_per_dst=[2712, 2582, 2564, 2676, 2580, 2618, 2652, 2506] combine_rows_per_src=[5720, 5568, 5558, 5592, 5554, 5538, 5554, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44458 payload_rows_remote=38998 dup_rows=21078 dup_rows_remote=18600 dedup_ratio=0.321625 router_tokens_per_expert=[518, 533, 476, 505, 478, 553, 550, 533, 548, 542, 517, 528, 526, 511, 510, 503, 496, 531, 480, 533, 504, 490, 518, 524, 513, 476, 497, 479, 498, 485, 528, 498, 526, 521, 515, 523, 563, 494, 518, 486, 513, 467, 512, 507, 479, 515, 496, 480, 524, 514, 541, 544, 525, 498, 550, 506, 531, 495, 489, 508, 534, 487, 505, 521] payload_rows_per_dst=[5624, 5614, 5498, 5416, 5694, 5460, 5630, 5522] dup_rows_per_dst=[2668, 2756, 2654, 2532, 2598, 2478, 2774, 2618] combine_rows_per_src=[5624, 5614, 5498, 5416, 5694, 5460, 5630, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279566336 dispatch_meta_bytes=230032 combine_payload_bytes=279566336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279796368 ingress_bytes=389666816 link_bytes=389666816 token_bytes=559362704 weight_movement_bytes=110100480 total_nvlink_bytes=669463184 weight_to_token_ratio=0.196832 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776946822e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280039424 dispatch_meta_bytes=230144 combine_payload_bytes=280039424 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280269568 ingress_bytes=500240384 link_bytes=500240384 token_bytes=560308992 weight_movement_bytes=220200960 total_nvlink_bytes=780509952 weight_to_token_ratio=0.392999 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.700003032e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44546 payload_rows_remote=38988 dup_rows=20990 dup_rows_remote=18216 dedup_ratio=0.320282 router_tokens_per_expert=[512, 506, 531, 524, 521, 496, 560, 513, 498, 501, 456, 488, 516, 529, 483, 517, 476, 513, 561, 573, 568, 468, 527, 519, 521, 484, 529, 466, 501, 540, 517, 504, 517, 507, 548, 500, 540, 482, 501, 499, 513, 496, 483, 502, 466, 542, 519, 513, 497, 531, 525, 509, 521, 535, 511, 537, 499, 502, 490, 521, 542, 510, 494, 498] payload_rows_per_dst=[5644, 5494, 5638, 5566, 5580, 5490, 5558, 5576] dup_rows_per_dst=[2682, 2482, 2772, 2558, 2608, 2578, 2774, 2536] combine_rows_per_src=[5644, 5494, 5638, 5566, 5580, 5490, 5558, 5576] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280111104 dispatch_meta_bytes=229544 combine_payload_bytes=280111104 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280340648 ingress_bytes=500312064 link_bytes=500312064 token_bytes=560451752 weight_movement_bytes=220200960 total_nvlink_bytes=780652712 weight_to_token_ratio=0.392899 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701249665e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5576158248 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=2789225512 ingress_bytes=4548540416 link_bytes=4548540416 total_nvlink_bytes=7337765928 weight_to_token_ratio=0.315918 prefetch_share_of_ingress=0.193645 dedup_ratio=0.321817 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.910659890e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=279623680 dispatch_meta_bytes=229664 combine_payload_bytes=279623680 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=279853344 ingress_bytes=609925120 link_bytes=609925120 token_bytes=559477024 weight_movement_bytes=330301440 total_nvlink_bytes=889778464 weight_to_token_ratio=0.590375 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.051471 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060760099e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44668 payload_rows_remote=39090 dup_rows=20868 dup_rows_remote=18374 dedup_ratio=0.318420 router_tokens_per_expert=[508, 509, 528, 534, 502, 525, 522, 530, 476, 516, 515, 508, 520, 495, 527, 524, 498, 515, 507, 490, 507, 491, 538, 515, 495, 499, 508, 509, 513, 492, 516, 504, 534, 498, 518, 545, 559, 535, 480, 518, 534, 493, 499, 529, 512, 482, 505, 499, 538, 487, 489, 507, 535, 487, 526, 509, 505, 497, 551, 517, 504, 554, 490, 496] payload_rows_per_dst=[5604, 5596, 5544, 5578, 5686, 5518, 5582, 5560] dup_rows_per_dst=[2712, 2566, 2578, 2494, 2688, 2588, 2574, 2668] combine_rows_per_src=[5604, 5596, 5544, 5578, 5686, 5518, 5582, 5560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278046720 dispatch_meta_bytes=227736 combine_payload_bytes=278046720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278274456 ingress_bytes=388147200 link_bytes=388147200 token_bytes=556321176 weight_movement_bytes=110100480 total_nvlink_bytes=666421656 weight_to_token_ratio=0.197908 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750518200e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5589406320 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2795850352 ingress_bytes=4665264128 link_bytes=4665264128 total_nvlink_bytes=7461114480 weight_to_token_ratio=0.334867 prefetch_share_of_ingress=0.236001 dedup_ratio=0.320743 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.113661623e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280111104 dispatch_meta_bytes=229544 combine_payload_bytes=280111104 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280340648 ingress_bytes=500312064 link_bytes=500312064 token_bytes=560451752 weight_movement_bytes=220200960 total_nvlink_bytes=780652712 weight_to_token_ratio=0.392899 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.701249665e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=230392 combine_payload_bytes=279537664 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279768056 ingress_bytes=389638144 link_bytes=389638144 token_bytes=559305720 weight_movement_bytes=110100480 total_nvlink_bytes=669406200 weight_to_token_ratio=0.196852 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776448169e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=228816 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279694800 ingress_bytes=499666944 link_bytes=499666944 token_bytes=559160784 weight_movement_bytes=220200960 total_nvlink_bytes=779361744 weight_to_token_ratio=0.393806 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.690029967e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44448 payload_rows_remote=38790 dup_rows=21088 dup_rows_remote=18144 dedup_ratio=0.321777 router_tokens_per_expert=[506, 483, 484, 496, 481, 524, 509, 523, 486, 516, 484, 516, 528, 557, 502, 517, 525, 508, 528, 503, 525, 545, 481, 518, 523, 505, 489, 499, 499, 557, 514, 508, 521, 527, 541, 475, 492, 462, 508, 502, 519, 518, 511, 496, 524, 498, 483, 517, 552, 529, 543, 500, 492, 532, 504, 541, 525, 511, 513, 496, 546, 485, 528, 538] payload_rows_per_dst=[5244, 5444, 5618, 5646, 5540, 5474, 5824, 5658] dup_rows_per_dst=[2124, 2450, 2648, 2728, 2516, 2658, 3020, 2944] combine_rows_per_src=[5244, 5444, 5618, 5646, 5540, 5474, 5824, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5589406320 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2795850352 ingress_bytes=4665264128 link_bytes=4665264128 total_nvlink_bytes=7461114480 weight_to_token_ratio=0.334867 prefetch_share_of_ingress=0.236001 dedup_ratio=0.320743 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.113661623e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5591673912 prefetch_bytes=880803840 reduce_bytes=660602880 weight_movement_bytes=1541406720 egress_bytes=2796985400 ingress_bytes=4336095232 link_bytes=4336095232 total_nvlink_bytes=7133080632 weight_to_token_ratio=0.275661 prefetch_share_of_ingress=0.203133 dedup_ratio=0.320596 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=6 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.541182774e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44638 payload_rows_remote=39038 dup_rows=20898 dup_rows_remote=18368 dedup_ratio=0.318878 router_tokens_per_expert=[565, 504, 500, 507, 508, 531, 485, 494, 520, 516, 513, 527, 522, 482, 571, 541, 494, 531, 510, 471, 511, 496, 501, 551, 520, 536, 517, 529, 538, 546, 494, 535, 490, 475, 500, 548, 494, 495, 516, 486, 500, 524, 509, 484, 496, 556, 529, 514, 527, 512, 534, 492, 494, 510, 538, 472, 510, 530, 502, 487, 524, 475, 483, 496] payload_rows_per_dst=[5548, 5626, 5600, 5690, 5496, 5642, 5550, 5486] dup_rows_per_dst=[2640, 2758, 2530, 2740, 2512, 2582, 2608, 2528] combine_rows_per_src=[5548, 5626, 5600, 5690, 5496, 5642, 5550, 5486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44458 payload_rows_remote=38998 dup_rows=21078 dup_rows_remote=18600 dedup_ratio=0.321625 router_tokens_per_expert=[518, 533, 476, 505, 478, 553, 550, 533, 548, 542, 517, 528, 526, 511, 510, 503, 496, 531, 480, 533, 504, 490, 518, 524, 513, 476, 497, 479, 498, 485, 528, 498, 526, 521, 515, 523, 563, 494, 518, 486, 513, 467, 512, 507, 479, 515, 496, 480, 524, 514, 541, 544, 525, 498, 550, 506, 531, 495, 489, 508, 534, 487, 505, 521] payload_rows_per_dst=[5624, 5614, 5498, 5416, 5694, 5460, 5630, 5522] dup_rows_per_dst=[2668, 2756, 2654, 2532, 2598, 2478, 2774, 2618] combine_rows_per_src=[5624, 5614, 5498, 5416, 5694, 5460, 5630, 5522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280197120 dispatch_meta_bytes=229856 combine_payload_bytes=280197120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280426976 ingress_bytes=390297600 link_bytes=390297600 token_bytes=560624096 weight_movement_bytes=110100480 total_nvlink_bytes=670724576 weight_to_token_ratio=0.196389 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787917193e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5594396536 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2798346104 ingress_bytes=4777859072 link_bytes=4777859072 total_nvlink_bytes=7576205176 weight_to_token_ratio=0.354249 prefetch_share_of_ingress=0.207395 dedup_ratio=0.320505 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.309482749e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44668 payload_rows_remote=39090 dup_rows=20868 dup_rows_remote=18374 dedup_ratio=0.318420 router_tokens_per_expert=[508, 509, 528, 534, 502, 525, 522, 530, 476, 516, 515, 508, 520, 495, 527, 524, 498, 515, 507, 490, 507, 491, 538, 515, 495, 499, 508, 509, 513, 492, 516, 504, 534, 498, 518, 545, 559, 535, 480, 518, 534, 493, 499, 529, 512, 482, 505, 499, 538, 487, 489, 507, 535, 487, 526, 509, 505, 497, 551, 517, 504, 554, 490, 496] payload_rows_per_dst=[5604, 5596, 5544, 5578, 5686, 5518, 5582, 5560] dup_rows_per_dst=[2712, 2566, 2578, 2494, 2688, 2588, 2574, 2668] combine_rows_per_src=[5604, 5596, 5544, 5578, 5686, 5518, 5582, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44428 payload_rows_remote=38846 dup_rows=21108 dup_rows_remote=18364 dedup_ratio=0.322083 router_tokens_per_expert=[513, 530, 472, 504, 549, 545, 521, 488, 551, 540, 529, 531, 448, 528, 510, 575, 497, 493, 507, 501, 483, 532, 497, 481, 542, 532, 508, 503, 526, 487, 506, 499, 513, 514, 532, 530, 521, 508, 524, 521, 498, 508, 520, 483, 513, 495, 516, 503, 500, 497, 535, 526, 523, 527, 485, 509, 514, 494, 511, 476, 529, 513, 527, 475] payload_rows_per_dst=[5582, 5690, 5482, 5580, 5582, 5444, 5592, 5476] dup_rows_per_dst=[2662, 2734, 2500, 2626, 2744, 2628, 2612, 2602] combine_rows_per_src=[5582, 5690, 5482, 5580, 5582, 5444, 5592, 5476] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44696 payload_rows_remote=39052 dup_rows=20840 dup_rows_remote=18114 dedup_ratio=0.317993 router_tokens_per_expert=[544, 540, 517, 509, 507, 514, 553, 480, 551, 534, 480, 473, 522, 502, 511, 508, 525, 541, 502, 558, 529, 501, 506, 518, 492, 501, 547, 517, 512, 479, 515, 494, 506, 467, 525, 511, 520, 512, 539, 523, 462, 458, 508, 514, 493, 484, 501, 491, 477, 494, 509, 495, 533, 524, 550, 543, 528, 501, 490, 523, 525, 505, 525, 550] payload_rows_per_dst=[5644, 5582, 5640, 5572, 5624, 5806, 5220, 5608] dup_rows_per_dst=[2726, 2594, 2712, 2542, 2534, 3034, 2012, 2686] combine_rows_per_src=[5644, 5582, 5640, 5572, 5624, 5806, 5220, 5608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5569186224 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=2785737136 ingress_bytes=4324855808 link_bytes=4324855808 total_nvlink_bytes=7110592944 weight_to_token_ratio=0.276774 prefetch_share_of_ingress=0.229118 dedup_ratio=0.321884 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.521635567e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5594396536 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2798346104 ingress_bytes=4777859072 link_bytes=4777859072 total_nvlink_bytes=7576205176 weight_to_token_ratio=0.354249 prefetch_share_of_ingress=0.207395 dedup_ratio=0.320505 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.309482749e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278046720 dispatch_meta_bytes=227736 combine_payload_bytes=278046720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278274456 ingress_bytes=388147200 link_bytes=388147200 token_bytes=556321176 weight_movement_bytes=110100480 total_nvlink_bytes=666421656 weight_to_token_ratio=0.197908 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.750518200e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279824384 dispatch_meta_bytes=229624 combine_payload_bytes=279824384 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280054008 ingress_bytes=500025344 link_bytes=500025344 token_bytes=559878392 weight_movement_bytes=220200960 total_nvlink_bytes=780079352 weight_to_token_ratio=0.393301 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.696263133e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5585622176 prefetch_bytes=770703360 reduce_bytes=1431306240 weight_movement_bytes=2202009600 egress_bytes=2793958560 ingress_bytes=4993673216 link_bytes=4993673216 total_nvlink_bytes=7787631776 weight_to_token_ratio=0.394228 prefetch_share_of_ingress=0.154336 dedup_ratio=0.320947 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=13 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.684819041e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279537664 dispatch_meta_bytes=230392 combine_payload_bytes=279537664 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279768056 ingress_bytes=389638144 link_bytes=389638144 token_bytes=559305720 weight_movement_bytes=110100480 total_nvlink_bytes=669406200 weight_to_token_ratio=0.196852 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.776448169e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280197120 dispatch_meta_bytes=229856 combine_payload_bytes=280197120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280426976 ingress_bytes=390297600 link_bytes=390297600 token_bytes=560624096 weight_movement_bytes=110100480 total_nvlink_bytes=670724576 weight_to_token_ratio=0.196389 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787917193e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278448128 dispatch_meta_bytes=228840 combine_payload_bytes=278448128 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278676968 ingress_bytes=498649088 link_bytes=498649088 token_bytes=557125096 weight_movement_bytes=220200960 total_nvlink_bytes=777326056 weight_to_token_ratio=0.395245 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.672327777e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279924736 dispatch_meta_bytes=228664 combine_payload_bytes=279924736 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280153400 ingress_bytes=610226176 link_bytes=610226176 token_bytes=560078136 weight_movement_bytes=330301440 total_nvlink_bytes=890379576 weight_to_token_ratio=0.589742 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061283685e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5569186224 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=2785737136 ingress_bytes=4324855808 link_bytes=4324855808 total_nvlink_bytes=7110592944 weight_to_token_ratio=0.276774 prefetch_share_of_ingress=0.229118 dedup_ratio=0.321884 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.521635567e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5578940064 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=2790616736 ingress_bytes=4660031488 link_bytes=4660031488 total_nvlink_bytes=7450648224 weight_to_token_ratio=0.335495 prefetch_share_of_ingress=0.236266 dedup_ratio=0.321365 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.104561201e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5591673912 prefetch_bytes=880803840 reduce_bytes=660602880 weight_movement_bytes=1541406720 egress_bytes=2796985400 ingress_bytes=4336095232 link_bytes=4336095232 total_nvlink_bytes=7133080632 weight_to_token_ratio=0.275661 prefetch_share_of_ingress=0.203133 dedup_ratio=0.320596 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=6 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.541182774e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5585622176 prefetch_bytes=770703360 reduce_bytes=1431306240 weight_movement_bytes=2202009600 egress_bytes=2793958560 ingress_bytes=4993673216 link_bytes=4993673216 total_nvlink_bytes=7787631776 weight_to_token_ratio=0.394228 prefetch_share_of_ingress=0.154336 dedup_ratio=0.320947 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=13 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=12319 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.684819041e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5576158248 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=2789225512 ingress_bytes=4548540416 link_bytes=4548540416 total_nvlink_bytes=7337765928 weight_to_token_ratio=0.315918 prefetch_share_of_ingress=0.193645 dedup_ratio=0.321817 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.910659890e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5576268912 prefetch_bytes=880803840 reduce_bytes=1541406720 weight_movement_bytes=2422210560 egress_bytes=2789278832 ingress_bytes=5209200640 link_bytes=5209200640 total_nvlink_bytes=7998479472 weight_to_token_ratio=0.434378 prefetch_share_of_ingress=0.169086 dedup_ratio=0.320901 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=14 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.059656679e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none - [2026-08-26 16:58:55] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 363.9 | throughput per GPU (TFLOP/s/GPU): 339.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.428962E+00 | loss scale: 1.0 | grad norm: 3.375 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-26 16:58:55 -[rank0]:[W826 16:58:56.908633964 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W826 16:58:57.904227803 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W826 16:58:57.921290484 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W826 16:58:58.052372833 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W826 16:58:58.469925490 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W826 16:58:58.487059907 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W826 16:58:58.579259445 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W826 16:58:58.741697318 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0826 16:59:00.009000 292582 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 16:59:00.010000 292582 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 292675 closing signal SIGTERM -W0826 16:59:00.010000 292582 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 292676 closing signal SIGTERM -W0826 16:59:00.012000 292582 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 292677 closing signal SIGTERM -W0826 16:59:00.014000 292582 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 292678 closing signal SIGTERM -W0826 16:59:00.015000 292582 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 292679 closing signal SIGTERM -W0826 16:59:00.017000 292582 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 292680 closing signal SIGTERM -W0826 16:59:00.018000 292582 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 292681 closing signal SIGTERM -W0826 16:59:00.019000 292582 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 292682 closing signal SIGTERM + [2026-08-26 18:11:00] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 359.1 | throughput per GPU (TFLOP/s/GPU): 343.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.428049E+00 | loss scale: 1.0 | grad norm: 3.382 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 18:11:00 +[rank0]:[W826 18:11:02.317447450 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 18:11:03.183106759 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 18:11:03.222062778 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 18:11:03.235686953 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 18:11:03.720540711 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 18:11:03.721292721 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 18:11:03.748824943 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 18:11:03.892298065 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 18:11:05.703000 465016 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 18:11:05.704000 465016 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 465109 closing signal SIGTERM +W0826 18:11:05.704000 465016 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 465110 closing signal SIGTERM +W0826 18:11:05.707000 465016 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 465111 closing signal SIGTERM +W0826 18:11:05.709000 465016 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 465112 closing signal SIGTERM +W0826 18:11:05.710000 465016 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 465113 closing signal SIGTERM +W0826 18:11:05.713000 465016 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 465114 closing signal SIGTERM +W0826 18:11:05.714000 465016 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 465115 closing signal SIGTERM +W0826 18:11:05.716000 465016 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 465116 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -3537,4 +3537,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 292582 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 465016 got signal: 15 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv index 876e9048c13adc821ba74ea48fd2c716fefd26aa..dd98d4846bf79459cd439ad6d1e91e18d4e72473 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +++ b/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -3,3 +3,5 @@ QWEN3_235B moonep natural none OOM 182589 1 0 /workspace/tierb_study_20260826_1 QWEN3_30B moonep natural none OOM 175959 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1 DSV2_LITE_L24 moonep natural none OK 157151 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 DSV2_LITE_L24 moonep forced none OK 157151 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1 +QWEN2_57B moonep natural none OK 141555 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 +QWEN2_57B moonep forced none OK 141555 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv index 17ac23c03bcf2c15cb45c2aa893e0ed1f8dd4c09..38f2fcee491b59a25cdbb82912047e9b969573af 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +++ b/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -2,3 +2,4 @@ model backend status selected_capacity required_capacity drop_rate_pct peak_mib QWEN3_235B moonep DOES_NOT_FIT 182589 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1 QWEN3_30B moonep DOES_NOT_FIT 175959 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1 DSV2_LITE_L24 moonep SELECTED none 0 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 +QWEN2_57B moonep SELECTED none 0 141555 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json b/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json index 0f558614b98163ee4942baf2042be107ee76a03d..146fa14f8f6f999f36cbfdbb19484499159ddc4e 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json +++ b/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json @@ -233,7 +233,7 @@ "capacity": "none", "capacity_mode": "none", "cold_expert_fraction_max": 0.796875, - "cold_expert_fraction_mean": 0.3940625, + "cold_expert_fraction_mean": 0.394375, "cold_expert_fraction_min": 0.0, "cold_expert_fraction_n": 100, "commit": "35ae6e7f", @@ -245,8 +245,8 @@ "global_batch_size": "8", "global_snapshots": 100, "hottest_expert_token_fraction_max": 1.0, - "hottest_expert_token_fraction_mean": 0.923837890625, - "hottest_expert_token_fraction_min": 0.238525390625, + "hottest_expert_token_fraction_mean": 0.9259124755859375, + "hottest_expert_token_fraction_min": 0.24102783203125, "hottest_expert_token_fraction_n": 100, "incomplete_snapshots": 0, "malformed_records": 0, @@ -254,77 +254,77 @@ "moonep_bytes_global_snapshots": 100, "moonep_bytes_incomplete_snapshots": 0, "moonep_bytes_malformed_records": 0, - "moonep_global_combine_payload_bytes_max": 3287474176.0, - "moonep_global_combine_payload_bytes_mean": 2792054415.36, - "moonep_global_combine_payload_bytes_min": 2200748032.0, + "moonep_global_combine_payload_bytes_max": 3287316480.0, + "moonep_global_combine_payload_bytes_mean": 2782399836.16, + "moonep_global_combine_payload_bytes_min": 2196246528.0, "moonep_global_combine_payload_bytes_n": 100, - "moonep_global_dispatch_payload_bytes_max": 3287474176.0, - "moonep_global_dispatch_payload_bytes_mean": 2792054415.36, - "moonep_global_dispatch_payload_bytes_min": 2200748032.0, + "moonep_global_dispatch_payload_bytes_max": 3287316480.0, + "moonep_global_dispatch_payload_bytes_mean": 2782399836.16, + "moonep_global_dispatch_payload_bytes_min": 2196246528.0, "moonep_global_dispatch_payload_bytes_n": 100, "moonep_global_prefetch_bytes_max": 990904320.0, - "moonep_global_prefetch_bytes_mean": 826854604.8, + "moonep_global_prefetch_bytes_mean": 824652595.2, "moonep_global_prefetch_bytes_min": 660602880.0, "moonep_global_prefetch_bytes_n": 100, "moonep_global_reduce_bytes_max": 990904320.0, - "moonep_global_reduce_bytes_mean": 826854604.8, + "moonep_global_reduce_bytes_mean": 824652595.2, "moonep_global_reduce_bytes_min": 660602880.0, "moonep_global_reduce_bytes_n": 100, - "moonep_global_token_bytes_max": 6576783336.0, - "moonep_global_token_bytes_mean": 5585911141.52, - "moonep_global_token_bytes_min": 4403305176.0, + "moonep_global_token_bytes_max": 6576467936.0, + "moonep_global_token_bytes_mean": 5566600373.52, + "moonep_global_token_bytes_min": 4394285176.0, "moonep_global_token_bytes_n": 100, - "moonep_global_total_nvlink_bytes_max": 8345010168.0, - "moonep_global_total_nvlink_bytes_mean": 7239620351.12, - "moonep_global_total_nvlink_bytes_min": 5944711896.0, + "moonep_global_total_nvlink_bytes_max": 8373193576.0, + "moonep_global_total_nvlink_bytes_mean": 7215905563.92, + "moonep_global_total_nvlink_bytes_min": 5935691896.0, "moonep_global_total_nvlink_bytes_n": 100, "moonep_global_weight_movement_bytes_max": 1981808640.0, - "moonep_global_weight_movement_bytes_mean": 1653709209.6, + "moonep_global_weight_movement_bytes_mean": 1649305190.4, "moonep_global_weight_movement_bytes_min": 1321205760.0, "moonep_global_weight_movement_bytes_n": 100, - "moonep_global_weight_to_token_ratio_max": 0.4264504160968184, - "moonep_global_weight_to_token_ratio_mean": 0.3001723215454669, - "moonep_global_weight_to_token_ratio_min": 0.20088935464362695, + "moonep_global_weight_to_token_ratio_max": 0.42643213883204506, + "moonep_global_weight_to_token_ratio_mean": 0.30020057542506196, + "moonep_global_weight_to_token_ratio_min": 0.2008989890709626, "moonep_global_weight_to_token_ratio_n": 100, "moonep_remote_demand_max": 2.0, - "moonep_remote_demand_mean": 0.93875, + "moonep_remote_demand_mean": 0.93625, "moonep_remote_demand_min": 0.0, "moonep_remote_demand_n": 800, "moonep_slot_utilization_max": 0.25, - "moonep_slot_utilization_mean": 0.11734375, + "moonep_slot_utilization_mean": 0.11703125, "moonep_slot_utilization_min": 0.0, "moonep_slot_utilization_n": 800, "moonep_unused_prefetch_slots_max": 16.0, - "moonep_unused_prefetch_slots_mean": 14.1225, + "moonep_unused_prefetch_slots_mean": 14.1275, "moonep_unused_prefetch_slots_min": 12.0, "moonep_unused_prefetch_slots_n": 800, "num_experts": "64", "num_layers": "10", - "post_physical_rank_cv_max": 0.19752362049364244, - "post_physical_rank_cv_mean": 0.08253319288939637, - "post_physical_rank_cv_min": 0.0001808697752907129, + "post_physical_rank_cv_max": 0.17539178027248586, + "post_physical_rank_cv_mean": 0.08677042581890185, + "post_physical_rank_cv_min": 0.00022118498628126193, "post_physical_rank_cv_n": 100, - "post_physical_rank_imax_max": 1.2480128708509701, - "post_physical_rank_imax_mean": 1.1014610244938725, - "post_physical_rank_imax_min": 1.0002403836981697, + "post_physical_rank_imax_max": 1.268348258961792, + "post_physical_rank_imax_mean": 1.1049963202647002, + "post_physical_rank_imax_min": 1.000290001068425, "post_physical_rank_imax_n": 100, - "pre_home_rank_cv_max": 1.3115485671633254, - "pre_home_rank_cv_mean": 0.6864519759847463, - "pre_home_rank_cv_min": 0.05302373213438137, + "pre_home_rank_cv_max": 1.3100187973257966, + "pre_home_rank_cv_mean": 0.6851584836928786, + "pre_home_rank_cv_min": 0.11244839305815055, "pre_home_rank_cv_n": 100, - "pre_home_rank_imax_max": 3.8150634765625, - "pre_home_rank_imax_mean": 2.2216781616210937, - "pre_home_rank_imax_min": 1.113861083984375, + "pre_home_rank_imax_max": 3.994415283203125, + "pre_home_rank_imax_mean": 2.235577392578125, + "pre_home_rank_imax_min": 1.23724365234375, "pre_home_rank_imax_n": 100, - "raw_expert_cv_max": 2.6451862448859087, - "raw_expert_cv_mean": 2.182940410315874, - "raw_expert_cv_min": 0.5018423789492765, + "raw_expert_cv_max": 2.6451977945600005, + "raw_expert_cv_mean": 2.180711127594004, + "raw_expert_cv_min": 0.49806402214302137, "raw_expert_cv_n": 100, "raw_expert_imax_ceiling": 8.0, "raw_expert_imax_hit_ceiling": 1, "raw_expert_imax_max": 8.0, - "raw_expert_imax_mean": 7.390703125, - "raw_expert_imax_min": 1.908203125, + "raw_expert_imax_mean": 7.4072998046875, + "raw_expert_imax_min": 1.92822265625, "raw_expert_imax_n": 100, "records": { "moonep_bytes": 800, diff --git a/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv b/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv index f13e65dd141bb8f9d7881643bc80b1de35d66d1b..9c2e7cb7c9281532f213ef03041a0b2acb3286a0 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv +++ b/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv @@ -1,5 +1,5 @@ run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records moonep_bytes_global_snapshots moonep_bytes_incomplete_snapshots moonep_bytes_malformed_records moonep_global_combine_payload_bytes_max moonep_global_combine_payload_bytes_mean moonep_global_combine_payload_bytes_min moonep_global_combine_payload_bytes_n moonep_global_dispatch_payload_bytes_max moonep_global_dispatch_payload_bytes_mean moonep_global_dispatch_payload_bytes_min moonep_global_dispatch_payload_bytes_n moonep_global_prefetch_bytes_max moonep_global_prefetch_bytes_mean moonep_global_prefetch_bytes_min moonep_global_prefetch_bytes_n moonep_global_reduce_bytes_max moonep_global_reduce_bytes_mean moonep_global_reduce_bytes_min moonep_global_reduce_bytes_n moonep_global_token_bytes_max moonep_global_token_bytes_mean moonep_global_token_bytes_min moonep_global_token_bytes_n moonep_global_total_nvlink_bytes_max moonep_global_total_nvlink_bytes_mean moonep_global_total_nvlink_bytes_min moonep_global_total_nvlink_bytes_n moonep_global_weight_movement_bytes_max moonep_global_weight_movement_bytes_mean moonep_global_weight_movement_bytes_min moonep_global_weight_movement_bytes_n moonep_global_weight_to_token_ratio_max moonep_global_weight_to_token_ratio_mean moonep_global_weight_to_token_ratio_min moonep_global_weight_to_token_ratio_n moonep_remote_demand_max moonep_remote_demand_mean moonep_remote_demand_min moonep_remote_demand_n moonep_slot_utilization_max moonep_slot_utilization_mean moonep_slot_utilization_min moonep_slot_utilization_n moonep_unused_prefetch_slots_max moonep_unused_prefetch_slots_mean moonep_unused_prefetch_slots_min moonep_unused_prefetch_slots_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 natural DSV2_LITE_L24 moonep 64 6 8 24 23 1 8192 8 none none 0 0 moonep_backend_token_routing_analysis 35ae6e7f seq_aux_loss 1e-3 0 0.578125 0.15625 0.0 230 230 0.999969482421875 0.7797577567722487 0.162750244140625 230 0 0 230 0 0 2592899072.0 2282338571.130435 2026799104.0 230 2592899072.0 2282338571.130435 2026799104.0 230 346030080.0 252601958.4 242221056.0 230 346030080.0 252601958.4 242221056.0 230 5188482536.0 4567339392.869565 4056064752.0 230 5672924648.0 5072543309.669565 4540506864.0 230 692060160.0 505203916.8 484442112.0 230 0.14851639788299403 0.11096021713514241 0.09336874676530664 230 3.0 0.9125 0.0 1840 0.375 0.1140625 0.0 1840 16.0 14.175 10.0 1840 0.24644338469771185 0.0866314272142518 0.016159622971429973 230 1.2001146488570558 1.1121655106296537 1.0188653892666146 230 1.3626457501059894 0.6592086328754161 0.06474057713157837 230 4.328043619791667 2.2370494897814766 1.1056925455729167 230 3.0868156557140782 1.9632045302354437 0.37852206422070017 230 10.666666666666666 1 10.666341145833334 8.317416072237318 1.7360026041666667 230 DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1 forced DSV2_LITE_L24 moonep 64 6 8 24 23 1 8192 8 none none 1 0 moonep_backend_token_routing_analysis 35ae6e7f seq_aux_loss 1e-3 0 0.0 0.0 0.0 230 230 0.0982818603515625 0.09643501613451087 0.09539794921875 230 0 0 230 0 0 2133762048.0 2129465023.4434783 2124537856.0 230 2133762048.0 2129465023.4434783 2124537856.0 230 242221056.0 238309411.61739132 207618048.0 230 242221056.0 238309411.61739132 207618048.0 230 4270279408.0 4261681018.2608695 4251824656.0 230 4754721520.0 4738299841.495652 4667434440.0 230 484442112.0 476618823.23478264 415236096.0 230 0.11393746242954192 0.11183841293995254 0.09728537714366937 230 1.0 0.8608695652173913 0.0 1840 0.125 0.10760869565217392 0.0 1840 16.0 14.278260869565218 14.0 1840 0.0031667036922078573 0.0016501228109707747 0.0006791931370339787 230 1.006201201251668 1.0026117364992109 1.0009196904719393 230 0.006673533732490181 0.003923942735502633 0.0014708009315538319 230 1.0122884114583333 1.0061054616734602 1.00213623046875 230 0.015123587213954862 0.012068912140234289 0.008171875787738593 230 10.666666666666666 0 1.04833984375 1.0286401721014493 1.017578125 230 -QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 natural QWEN2_57B moonep 64 8 8 10 10 0 4096 8 none none 0 0 moonep_backend_token_routing_analysis 35ae6e7f aux_loss 1e-2 0 0.796875 0.3940625 0.0 100 100 1.0 0.923837890625 0.238525390625 100 0 0 100 0 0 3287474176.0 2792054415.36 2200748032.0 100 3287474176.0 2792054415.36 2200748032.0 100 990904320.0 826854604.8 660602880.0 100 990904320.0 826854604.8 660602880.0 100 6576783336.0 5585911141.52 4403305176.0 100 8345010168.0 7239620351.12 5944711896.0 100 1981808640.0 1653709209.6 1321205760.0 100 0.4264504160968184 0.3001723215454669 0.20088935464362695 100 2.0 0.93875 0.0 800 0.25 0.11734375 0.0 800 16.0 14.1225 12.0 800 0.19752362049364244 0.08253319288939637 0.0001808697752907129 100 1.2480128708509701 1.1014610244938725 1.0002403836981697 100 1.3115485671633254 0.6864519759847463 0.05302373213438137 100 3.8150634765625 2.2216781616210937 1.113861083984375 100 2.6451862448859087 2.182940410315874 0.5018423789492765 100 8.0 1 8.0 7.390703125 1.908203125 100 +QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 natural QWEN2_57B moonep 64 8 8 10 10 0 4096 8 none none 0 0 moonep_backend_token_routing_analysis 35ae6e7f aux_loss 1e-2 0 0.796875 0.394375 0.0 100 100 1.0 0.9259124755859375 0.24102783203125 100 0 0 100 0 0 3287316480.0 2782399836.16 2196246528.0 100 3287316480.0 2782399836.16 2196246528.0 100 990904320.0 824652595.2 660602880.0 100 990904320.0 824652595.2 660602880.0 100 6576467936.0 5566600373.52 4394285176.0 100 8373193576.0 7215905563.92 5935691896.0 100 1981808640.0 1649305190.4 1321205760.0 100 0.42643213883204506 0.30020057542506196 0.2008989890709626 100 2.0 0.93625 0.0 800 0.25 0.11703125 0.0 800 16.0 14.1275 12.0 800 0.17539178027248586 0.08677042581890185 0.00022118498628126193 100 1.268348258961792 1.1049963202647002 1.000290001068425 100 1.3100187973257966 0.6851584836928786 0.11244839305815055 100 3.994415283203125 2.235577392578125 1.23724365234375 100 2.6451977945600005 2.180711127594004 0.49806402214302137 100 8.0 1 8.0 7.4072998046875 1.92822265625 100 QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1 forced QWEN2_57B moonep 64 8 8 10 10 0 4096 8 none none 1 0 moonep_backend_token_routing_analysis 35ae6e7f aux_loss 1e-2 0 0.0 0.0 0.0 100 100 0.13189697265625 0.1293536376953125 0.1279296875 100 0 0 100 0 0 2236903424.0 2231590215.68 2225391616.0 100 2236903424.0 2231590215.68 2225391616.0 100 770703360.0 761895321.6 550502400.0 100 770703360.0 761895321.6 550502400.0 100 4475642264.0 4465014321.2 4452613880.0 100 6017048984.0 5988804964.4 5568474336.0 100 1541406720.0 1523790643.2 1101004800.0 100 0.34618019023019353 0.3412749588086127 0.2464493134486592 100 1.0 0.865 0.0 800 0.125 0.108125 0.0 800 16.0 14.27 14.0 800 0.004151405986167901 0.002179447588207395 0.0007203705850357968 100 1.0062412574645103 1.0033121152297226 1.0013429305103698 100 0.009250589744190636 0.004711613912774553 0.002294812301292466 100 1.015838623046875 1.00729736328125 1.002899169921875 100 0.018080370892918116 0.014541988425652842 0.011942398269672016 100 8.0 0 1.05517578125 1.0348291015625 1.0234375 100 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/results.tsv b/moonep/tierbprobe/tierbprobe_gbs8/results.tsv index 49174012fc7dfb1d6930b85b0d7f31716d9e4e07..5fc653d094d76feb79a6e8619d667e2f367d1243 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/results.tsv +++ b/moonep/tierbprobe/tierbprobe_gbs8/results.tsv @@ -3,3 +3,5 @@ QWEN3_235B moonep matched 16 8192 8 none none 1 OOM NA NA 182589 /workspace/tier QWEN3_30B moonep matched 16 4096 8 none none 1 OOM NA NA 175959 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1 DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK 799.34 793.30 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 DSV2_LITE_L24 moonep matched 8 8192 8 none none+forcebal 1 OK 792.51 789.10 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1 +QWEN2_57B moonep matched 8 4096 8 none none 1 OK 416.69 364.70 141555 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 +QWEN2_57B moonep matched 8 4096 8 none none+forcebal 1 OK 405.27 359.10 141555 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1